From d35031bfc5a58d9dca9f0907fc18c614186b089a Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Mon, 24 Feb 2025 11:20:37 +0100 Subject: [PATCH 01/91] :construction: start with bundleing cli tool --- executables/README.md | 6 ++++++ executables/vuegen.spec | 48 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+) create mode 100644 executables/README.md create mode 100644 executables/vuegen.spec diff --git a/executables/README.md b/executables/README.md new file mode 100644 index 0000000..82dcb45 --- /dev/null +++ b/executables/README.md @@ -0,0 +1,6 @@ +# Pyinstaller one folder executable + +``` +# from root of the project +pyinstaller -name vuegen src/vuegen/__main__.py +``` diff --git a/executables/vuegen.spec b/executables/vuegen.spec new file mode 100644 index 0000000..39fb810 --- /dev/null +++ b/executables/vuegen.spec @@ -0,0 +1,48 @@ +# -*- mode: python ; coding: utf-8 -*- +from PyInstaller.utils.hooks import collect_all + +datas = [] +binaries = [] +hiddenimports = [] + +a = Analysis( + ['src/vuegen/__main__.py'], + pathex=[], + binaries=binaries, + datas=datas, + hiddenimports=hiddenimports, + hookspath=[], + hooksconfig={}, + runtime_hooks=[], + excludes=[], + noarchive=False, + optimize=0, +) +pyz = PYZ(a.pure) + +exe = EXE( + pyz, + a.scripts, + [], + exclude_binaries=True, + name='vuegen', + debug=False, + bootloader_ignore_signals=False, + strip=False, + upx=True, + console=True, + disable_windowed_traceback=False, + argv_emulation=False, + target_arch=None, + codesign_identity=None, + entitlements_file=None, +) +coll = COLLECT( + exe, + a.binaries, + a.datas, + strip=False, + upx=True, + upx_exclude=[], + name='vuegen', +) From 41cdd53bb82e6fecaaf00e5a5f57d16db80a528a Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Mon, 24 Feb 2025 11:23:48 +0100 Subject: [PATCH 02/91] :sparkles: with pyvis added the basic example works using the cli --- executables/README.md | 4 +++- executables/vuegen.spec | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/executables/README.md b/executables/README.md index 82dcb45..f4a800c 100644 --- a/executables/README.md +++ b/executables/README.md @@ -1,6 +1,8 @@ # Pyinstaller one folder executable +- pyvis templates were not copied, so make these explicit (see [this](https://stackoverflow.com/a/72687433/9684872)) + ``` # from root of the project -pyinstaller -name vuegen src/vuegen/__main__.py +pyinstaller -D -w --collect-all pyvis -n vuegen src/vuegen/__main__.py ``` diff --git a/executables/vuegen.spec b/executables/vuegen.spec index 39fb810..e40addf 100644 --- a/executables/vuegen.spec +++ b/executables/vuegen.spec @@ -4,6 +4,8 @@ from PyInstaller.utils.hooks import collect_all datas = [] binaries = [] hiddenimports = [] +tmp_ret = collect_all('pyvis') +datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2] a = Analysis( ['src/vuegen/__main__.py'], From 47b924b8072ed0b68802d25a2a10f502ff024fca Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Mon, 24 Feb 2025 14:46:06 +0100 Subject: [PATCH 03/91] :construction::bug: streamlit not easy to run from within vuegen script - sys.exectuable is vuegen script, not the original Python Interpreter -> not easy to run `pthon -m streamlit run file.py` - manually starting and building streamlit_run command does not connect to browser atm --- executables/README.md | 24 +++++++++++-- executables/vuegen.spec | 5 ++- src/vuegen/streamlit_reportview.py | 56 +++++++++++++++++++++++++++--- 3 files changed, 78 insertions(+), 7 deletions(-) diff --git a/executables/README.md b/executables/README.md index f4a800c..b55f886 100644 --- a/executables/README.md +++ b/executables/README.md @@ -2,7 +2,27 @@ - pyvis templates were not copied, so make these explicit (see [this](https://stackoverflow.com/a/72687433/9684872)) -``` +```bash # from root of the project -pyinstaller -D -w --collect-all pyvis -n vuegen src/vuegen/__main__.py +pyinstaller -D --collect-all pyvis -n vuegen src/vuegen/__main__.py +# from this README folder +pyinstaller -D --collect-all pyvis --collect-all streamlit -n vuegen ../src/vuegen/__main__.py +``` + +## Pyinstaller options + +```bash +What to generate: + -D, --onedir Create a one-folder bundle containing an executable (default) + -F, --onefile Create a one-file bundled executable. + --specpath DIR Folder to store the generated spec file (default: current directory) + -n NAME, --name NAME Name to assign to the bundled app and spec file (default: first script's basename) +Windows and macOS specific options: + -c, --console, --nowindowed + Open a console window for standard i/o (default). On Windows this option has no effect if the first script is a + '.pyw' file. + -w, --windowed, --noconsole + Windows and macOS: do not provide a console window for standard i/o. On macOS this also triggers building a + macOS .app bundle. On Windows this option is automatically set if the first script is a '.pyw' file. This option + is ignored on *NIX systems. ``` diff --git a/executables/vuegen.spec b/executables/vuegen.spec index e40addf..da24c97 100644 --- a/executables/vuegen.spec +++ b/executables/vuegen.spec @@ -6,9 +6,12 @@ binaries = [] hiddenimports = [] tmp_ret = collect_all('pyvis') datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2] +tmp_ret = collect_all('streamlit') +datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2] + a = Analysis( - ['src/vuegen/__main__.py'], + ['../src/vuegen/__main__.py'], pathex=[], binaries=binaries, datas=datas, diff --git a/src/vuegen/streamlit_reportview.py b/src/vuegen/streamlit_reportview.py index 789c73a..1da8d9a 100644 --- a/src/vuegen/streamlit_reportview.py +++ b/src/vuegen/streamlit_reportview.py @@ -1,11 +1,35 @@ import os import subprocess +import sys from typing import List import pandas as pd +from streamlit.web.cli import _main_run as streamlit_run from . import report as r -from .utils import create_folder, is_url, generate_footer +from .utils import create_folder, generate_footer, is_url + + +def streamlit_run( + file, + args=None, + flag_options=None, + **kwargs +) -> None: + if args is None: + args = [] + + if flag_options is None: + flag_options = {} + + import streamlit.web.bootstrap as bootstrap + from streamlit.runtime.credentials import check_credentials + + bootstrap.load_config_options(flag_options=kwargs) + + check_credentials() + + bootstrap.run(file, False, args, flag_options) class StreamlitReportView(r.WebAppReportView): @@ -115,9 +139,33 @@ def run_report(self, output_dir: str = SECTIONS_DIR) -> None: The folder where the report was generated (default is SECTIONS_DIR). """ if self.streamlit_autorun: - self.report.logger.info(f"Running '{self.report.title}' {self.report_type} report.") + self.report.logger.info( + f"Running '{self.report.title}' {self.report_type} report." + ) + self.report.logger.debug( + f"Running Streamlit report from directory: {output_dir}" + ) + # command = [ + # sys.executable, # ! will be vuegen main script, not the Python Interpreter + # "-m", + # "streamlit", + # "run", + # os.path.join(output_dir, self.REPORT_MANAG_SCRIPT), + # ] + self.report.logger.debug(sys.executable) + self.report.logger.debug(sys.path) try: - subprocess.run(["streamlit", "run", os.path.join(output_dir, self.REPORT_MANAG_SCRIPT)], check=True) + # ! streamlit is not known in packaged app + # self.report.logger.debug(f"Running command: {' '.join(command)}") + # subprocess.run( + # command, + # check=True, + # ) + target_file = os.path.join(output_dir, self.REPORT_MANAG_SCRIPT) + self.report.logger.debug( + f"Running Streamlit report from file: {target_file}" + ) + streamlit_run(target_file) except KeyboardInterrupt: print("Streamlit process interrupted.") except subprocess.CalledProcessError as e: @@ -747,4 +795,4 @@ def _generate_component_imports(self, component: r.Component) -> List[str]: component_imports.append('df_index = 1') # Return the list of import statements - return component_imports \ No newline at end of file + return component_imports From d874d395c410d0d87fd0fafe4387c182ca2c3bb2 Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Wed, 26 Feb 2025 13:35:36 +0100 Subject: [PATCH 04/91] :sparkles: start streamlit without subprocess - runs from self-contained app --- src/vuegen/streamlit_reportview.py | 72 +++++++----------------------- 1 file changed, 17 insertions(+), 55 deletions(-) diff --git a/src/vuegen/streamlit_reportview.py b/src/vuegen/streamlit_reportview.py index 22561da..74d07b2 100644 --- a/src/vuegen/streamlit_reportview.py +++ b/src/vuegen/streamlit_reportview.py @@ -4,34 +4,12 @@ from typing import List import pandas as pd -from streamlit.web.cli import _main_run as streamlit_run +from streamlit.web import cli as stcli from . import report as r from .utils import create_folder, generate_footer, is_url -def streamlit_run( - file, - args=None, - flag_options=None, - **kwargs -) -> None: - if args is None: - args = [] - - if flag_options is None: - flag_options = {} - - import streamlit.web.bootstrap as bootstrap - from streamlit.runtime.credentials import check_credentials - - bootstrap.load_config_options(flag_options=kwargs) - - check_credentials() - - bootstrap.run(file, False, args, flag_options) - - class StreamlitReportView(r.WebAppReportView): """ A Streamlit-based implementation of the WebAppReportView abstract base class. @@ -184,44 +162,28 @@ def run_report(self, output_dir: str = SECTIONS_DIR) -> None: self.report.logger.debug( f"Running Streamlit report from directory: {output_dir}" ) - # command = [ - # sys.executable, # ! will be vuegen main script, not the Python Interpreter - # "-m", - # "streamlit", - # "run", - # os.path.join(output_dir, self.REPORT_MANAG_SCRIPT), - # ] - self.report.logger.debug(sys.executable) - self.report.logger.debug(sys.path) + # ! using pyinstaller: vuegen main script as executable, not the Python Interpreter + msg = f"{sys.executable = }" + self.report.logger.debug(msg) try: - # ! streamlit is not known in packaged app - # self.report.logger.debug(f"Running command: {' '.join(command)}") - # subprocess.run( - # command, - # check=True, - # ) + # ! streamlit command option is not known in packaged app target_file = os.path.join(output_dir, self.REPORT_MANAG_SCRIPT) self.report.logger.debug( f"Running Streamlit report from file: {target_file}" ) - streamlit_run(target_file) - self.report.logger.info( - f"Running '{self.report.title}' {self.report_type} report." - ) - try: - subprocess.run( - [ - "streamlit", - "run", - os.path.join(output_dir, self.REPORT_MANAG_SCRIPT), - ], - check=True, - ) + args = [ + "streamlit", + "run", + target_file, + ] + sys.argv = args + + sys.exit(stcli.main()) except KeyboardInterrupt: print("Streamlit process interrupted.") - except subprocess.CalledProcessError as e: - self.report.logger.error(f"Error running Streamlit report: {str(e)}") - raise + # except subprocess.CalledProcessError as e: + # self.report.logger.error(f"Error running Streamlit report: {str(e)}") + # raise else: # If autorun is False, print instructions for manual execution self.report.logger.info( @@ -1003,6 +965,6 @@ def _generate_component_imports(self, component: r.Component) -> List[str]: component_imports.append("df_index = 1") # Return the list of import statements - return component_imports + return component_imports return component_imports From 363e8bfdf1a240572569e3a3fa96aa868e3d8900 Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Wed, 26 Feb 2025 13:59:15 +0100 Subject: [PATCH 05/91] :bug: ensure that networking is properly enabled - network URL assigned. - localhost not on port 3000, but 8501 it seems --- executables/README.md | 8 ++++++++ src/vuegen/streamlit_reportview.py | 1 + 2 files changed, 9 insertions(+) diff --git a/executables/README.md b/executables/README.md index b55f886..39cbf66 100644 --- a/executables/README.md +++ b/executables/README.md @@ -26,3 +26,11 @@ Windows and macOS specific options: macOS .app bundle. On Windows this option is automatically set if the first script is a '.pyw' file. This option is ignored on *NIX systems. ``` + +## Using bundled executable + +try using basic example + +```bash +./dist/vuegen/vuegen -d ../docs/example_data/Basic_example_vuegen_demo_notebook -st_autorun +``` diff --git a/src/vuegen/streamlit_reportview.py b/src/vuegen/streamlit_reportview.py index 74d07b2..e25fe54 100644 --- a/src/vuegen/streamlit_reportview.py +++ b/src/vuegen/streamlit_reportview.py @@ -175,6 +175,7 @@ def run_report(self, output_dir: str = SECTIONS_DIR) -> None: "streamlit", "run", target_file, + "--global.developmentMode=false", ] sys.argv = args From 0c1d2797d1dbbb7c0ce91fb9784fa8b3ce1d916b Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Wed, 26 Feb 2025 15:24:33 +0100 Subject: [PATCH 06/91] :art: make Streamlit report aware of where it runs - keep also subprocess option in code for local development --- src/vuegen/streamlit_reportview.py | 37 ++++++++++++++++++++---------- 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/src/vuegen/streamlit_reportview.py b/src/vuegen/streamlit_reportview.py index e25fe54..2a156a7 100644 --- a/src/vuegen/streamlit_reportview.py +++ b/src/vuegen/streamlit_reportview.py @@ -28,6 +28,12 @@ def __init__( ): super().__init__(report=report, report_type=report_type) self.streamlit_autorun = streamlit_autorun + self.BUNDLED_EXECUTION = False + if getattr(sys, "frozen", False) and hasattr(sys, "_MEIPASS"): + self.report.logger.info("running in a PyInstaller bundle") + self.BUNDLED_EXECUTION = True + else: + self.report.logger.info("running in a normal Python process") def generate_report( self, output_dir: str = SECTIONS_DIR, static_dir: str = STATIC_FILES_DIR @@ -171,20 +177,27 @@ def run_report(self, output_dir: str = SECTIONS_DIR) -> None: self.report.logger.debug( f"Running Streamlit report from file: {target_file}" ) - args = [ - "streamlit", - "run", - target_file, - "--global.developmentMode=false", - ] - sys.argv = args - - sys.exit(stcli.main()) + if self.BUNDLED_EXECUTION: + args = [ + "streamlit", + "run", + target_file, + "--global.developmentMode=false", + ] + sys.argv = args + + sys.exit(stcli.main()) + else: + self.report.logger.debug("Run using subprocess.") + subprocess.run( + [sys.executable, "-m", "streamlit", "run", target_file], + check=True, + ) except KeyboardInterrupt: print("Streamlit process interrupted.") - # except subprocess.CalledProcessError as e: - # self.report.logger.error(f"Error running Streamlit report: {str(e)}") - # raise + except subprocess.CalledProcessError as e: + self.report.logger.error(f"Error running Streamlit report: {str(e)}") + raise else: # If autorun is False, print instructions for manual execution self.report.logger.info( From 1d23016151c9660a9a3bcd6637faed01eed7077f Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Wed, 26 Feb 2025 15:30:04 +0100 Subject: [PATCH 07/91] :bug: add st_aggrid as it is imported, not installed import st_aggrid but pip install streamlit-aggrid it is a "hidden" import as the main script is not using it (as pyvis or streamlit itself) --- executables/README.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/executables/README.md b/executables/README.md index 39cbf66..385c57f 100644 --- a/executables/README.md +++ b/executables/README.md @@ -3,10 +3,13 @@ - pyvis templates were not copied, so make these explicit (see [this](https://stackoverflow.com/a/72687433/9684872)) ```bash -# from root of the project -pyinstaller -D --collect-all pyvis -n vuegen src/vuegen/__main__.py # from this README folder -pyinstaller -D --collect-all pyvis --collect-all streamlit -n vuegen ../src/vuegen/__main__.py +pyinstaller -D --collect-all pyvis --collect-all streamlit --collect-all st_aggrid -n vuegen ../src/vuegen/__main__.py +``` + +```bash +# other pyinstaller options +--noconfirm # is used to avoid the prompt for overwriting the existing dist folder ``` ## Pyinstaller options From c8eac52df2643c463d1999c1b64891be9fc5588c Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Wed, 26 Feb 2025 15:54:50 +0100 Subject: [PATCH 08/91] :construction: start testing of bundeling for quarto - some form of binaries have to be passed, along addtional dependencies --- src/vuegen/quarto_reportview.py | 35 ++++++++++++++++++++++++--------- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/src/vuegen/quarto_reportview.py b/src/vuegen/quarto_reportview.py index 600174d..68725fb 100644 --- a/src/vuegen/quarto_reportview.py +++ b/src/vuegen/quarto_reportview.py @@ -1,5 +1,6 @@ import os import subprocess +import sys from pathlib import Path from typing import List @@ -20,6 +21,12 @@ class QuartoReportView(r.ReportView): def __init__(self, report: r.Report, report_type: r.ReportType): super().__init__(report=report, report_type=report_type) + self.BUNDLED_EXECUTION = False + if getattr(sys, "frozen", False) and hasattr(sys, "_MEIPASS"): + self.report.logger.info("running in a PyInstaller bundle") + self.BUNDLED_EXECUTION = True + else: + self.report.logger.info("running in a normal Python process") def generate_report( self, output_dir: str = BASE_DIR, static_dir: str = STATIC_FILES_DIR @@ -155,22 +162,32 @@ def run_report(self, output_dir: str = BASE_DIR) -> None: The folder where the report was generated (default is 'sections'). """ try: - subprocess.run( - ["quarto", "render", os.path.join(output_dir, f"{self.BASE_DIR}.qmd")], - check=True, - ) - if self.report_type == r.ReportType.JUPYTER: + if not self.BUNDLED_EXECUTION: subprocess.run( [ "quarto", - "convert", + "render", os.path.join(output_dir, f"{self.BASE_DIR}.qmd"), ], check=True, ) - self.report.logger.info( - f"'{self.report.title}' '{self.report_type}' report rendered" - ) + if self.report_type == r.ReportType.JUPYTER: + subprocess.run( + [ + "quarto", + "convert", + os.path.join(output_dir, f"{self.BASE_DIR}.qmd"), + ], + check=True, + ) + self.report.logger.info( + f"'{self.report.title}' '{self.report_type}' report rendered" + ) + else: + self.report.logger.info( + f"Quarto report '{self.report.title}' '{self.report_type}' cannot " + "be run in a PyInstaller bundle" + ) except subprocess.CalledProcessError as e: self.report.logger.error( f"Error running '{self.report.title}' {self.report_type} report: {str(e)}" From 2387f23ec1b3f13c055ae118f36a8215ec949cec Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Wed, 26 Feb 2025 16:12:03 +0100 Subject: [PATCH 09/91] :memo: add current spec file --- executables/vuegen.spec | 2 ++ 1 file changed, 2 insertions(+) diff --git a/executables/vuegen.spec b/executables/vuegen.spec index da24c97..463d64d 100644 --- a/executables/vuegen.spec +++ b/executables/vuegen.spec @@ -8,6 +8,8 @@ tmp_ret = collect_all('pyvis') datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2] tmp_ret = collect_all('streamlit') datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2] +tmp_ret = collect_all('st_aggrid') +datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2] a = Analysis( From 1fa37c8e65eaeaba427e5d3c2cf4e0f6e9dc693a Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Fri, 28 Feb 2025 11:30:28 +0100 Subject: [PATCH 10/91] :sparkles:: First simple working example of a GUI (streamlit local) - can start streamlit when not package as exectuable from GUI - use grid to define layout ToDo: Move to class-based layout --- gui/README.md | 20 ++++++ gui/app.py | 163 +++++++++++++++++++++++++++++++++++++++++++++++++ pyproject.toml | 2 +- 3 files changed, 184 insertions(+), 1 deletion(-) create mode 100644 gui/README.md create mode 100644 gui/app.py diff --git a/gui/README.md b/gui/README.md new file mode 100644 index 0000000..45aa2b6 --- /dev/null +++ b/gui/README.md @@ -0,0 +1,20 @@ +# VueGen GUI + +## Local execution of the GUI + +Install required dependencies from package + +```bash +pip install 'vuegen[gui]' +# or with local repo +pip install '.[gui]' +# or for local editable install +pip install -e '.[gui]' +``` + +Can be started locally with + +```bash +# from within gui directory +python app.py +``` diff --git a/gui/app.py b/gui/app.py new file mode 100644 index 0000000..864fa33 --- /dev/null +++ b/gui/app.py @@ -0,0 +1,163 @@ +"""GUI for vuegen command-line tool. + +usage: VueGen [-h] [-c CONFIG] [-dir DIRECTORY] [-rt REPORT_TYPE] + [-st_autorun] + +optional arguments: + -h, --help show this help message and exit + -c CONFIG, --config CONFIG + Path to the YAML configuration file. + -dir DIRECTORY, --directory DIRECTORY + Path to the directory from which the YAML + config will be inferred. + -rt REPORT_TYPE, --report_type REPORT_TYPE + Type of the report to generate (streamlit, + html, pdf, docx, odt, revealjs, pptx, or + jupyter). + -st_autorun, --streamlit_autorun + Automatically run the Streamlit app after + report generation. +""" + +import sys +import tkinter as tk + +import customtkinter + +from vuegen.__main__ import main +from vuegen.report import ReportType + +customtkinter.set_appearance_mode("system") +customtkinter.set_default_color_theme("dark-blue") + + +# callbacks +def create_run_vuegen(is_dir, config_path, report_type, run_streamlit): + def inner(): + args = ["vuegen"] + if is_dir: + args.append("--directory") + else: + args.append("--config") + args.append(config_path.get()) + args.append("--report_type") + args.append(report_type.get()) + if run_streamlit: + args.append("--streamlit_autorun") + print("args:", args) + sys.argv = args + main() # Call the main function from vuegen + + return inner + + +def optionmenu_callback(choice): + """Good for logging changes?""" + print("optionmenu dropdown clicked:", choice) + + +def radiobutton_event(value): + def radio_button_callback(): + print("radiobutton toggled, current value:", value.get()) + + return radio_button_callback + + +# Options + +# get list of report types from Enum +report_types = [report_type.value.lower() for report_type in ReportType] + + +# APP +app = customtkinter.CTk() +app.geometry("460x400") +app.title("VueGen GUI") + +########################################################################################## +# Config or directory input +ctk_label_config = customtkinter.CTkLabel( + app, + text="Add path to config file or directory. Select radio button accordingly", +) +ctk_label_config.grid(row=0, column=0, columnspan=2, padx=20, pady=20) +is_dir = tk.IntVar(value=1) +callback_radio_config = radiobutton_event(is_dir) +ctk_radio_config_0 = customtkinter.CTkRadioButton( + app, + text="Use config", + command=callback_radio_config, + variable=is_dir, + value=0, +) +ctk_radio_config_0.grid(row=1, column=0, padx=20, pady=2) +ctk_radio_config_1 = customtkinter.CTkRadioButton( + app, + text="Use dir", + command=callback_radio_config, + variable=is_dir, + value=1, +) +ctk_radio_config_1.grid(row=1, column=1, padx=20, pady=2) + +config_path = tk.StringVar() +config_path_entry = customtkinter.CTkEntry( + app, + width=400, + textvariable=config_path, +) +config_path_entry.grid(row=2, column=0, columnspan=2, padx=20, pady=10) + +########################################################################################## +# Report type dropdown +ctk_label_report = customtkinter.CTkLabel( + app, + text="Select type of report to generate (use streamlit for now)", +) +ctk_label_report.grid(row=3, column=0, columnspan=2, padx=20, pady=20) +report_type = tk.StringVar(value=report_types[0]) +report_dropdown = customtkinter.CTkOptionMenu( + app, + values=report_types, + variable=report_type, + command=optionmenu_callback, +) +report_dropdown.grid(row=4, column=0, columnspan=2, padx=20, pady=20) + +_report_type = report_dropdown.get() +print("report_type value:", _report_type) + +########################################################################################## +# Run Streamlit radio button +run_streamlit = tk.IntVar(value=0) +callback_radio_st_run = radiobutton_event(run_streamlit) +ctk_radio_st_autorun_1 = customtkinter.CTkRadioButton( + app, + text="autorun streamlit", + value=1, + variable=run_streamlit, + command=callback_radio_st_run, +) +ctk_radio_st_autorun_1.grid(row=5, column=0, padx=20, pady=20) +ctk_radio_st_autorun_0 = customtkinter.CTkRadioButton( + app, + text="skip starting streamlit", + value=0, + variable=run_streamlit, + command=callback_radio_st_run, +) +ctk_radio_st_autorun_0.grid(row=5, column=1, padx=20, pady=20) + +########################################################################################## +# Run VueGen button +run_vuegen = create_run_vuegen(is_dir, config_path, report_type, run_streamlit) +run_button = customtkinter.CTkButton( + app, + text="Run VueGen", + command=run_vuegen, +) +run_button.grid(row=6, column=0, columnspan=2, padx=20, pady=20) + +########################################################################################## +# Run the app in the mainloop +app.mainloop() diff --git a/pyproject.toml b/pyproject.toml index 0e0a006..0bcb820 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,7 +55,7 @@ build-backend = "poetry_dynamic_versioning.backend" # https://stackoverflow.com/a/60990574/9684872 [tool.poetry.extras] -docs = ["sphinx", "sphinx-book-theme", "myst-nb", "ipywidgets", "sphinx-new-tab-link", "jupytext"] +gui = ["customtkinter"] [tool.poetry.scripts] # https://python-poetry.org/docs/pyproject/#scripts From 3ee9a554174e91c78add7850707d9e4859d60604 Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Fri, 28 Feb 2025 13:01:38 +0100 Subject: [PATCH 11/91] :sparkles: add basic GUI command - creates relative large executable (740 MB) --- gui/README.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/gui/README.md b/gui/README.md index 45aa2b6..c71974e 100644 --- a/gui/README.md +++ b/gui/README.md @@ -18,3 +18,22 @@ Can be started locally with # from within gui directory python app.py ``` + +## Build executable GUI + +For now do not add the `--windowed` option, as it will not show the console output, +which is useful for debugging and especially terminating any running processes, e.g. +as the streamlit server and the GUI itself. + +```bash +# from this README folder +pyinstaller \ +-n vuegen_gui \ +--no-confirm \ +--onedir \ +--collect-all pyvis \ +--collect-all streamlit \ +--collect-all st_aggrid \ +--collect-all customtkinter \ +app.py +``` From c4925d6462b152150559d63095b2d3a67bf37ac8 Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Fri, 28 Feb 2025 13:04:11 +0100 Subject: [PATCH 12/91] :art: set default to running streamlit app directly --- gui/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gui/app.py b/gui/app.py index 864fa33..0b16eca 100644 --- a/gui/app.py +++ b/gui/app.py @@ -129,7 +129,7 @@ def radio_button_callback(): ########################################################################################## # Run Streamlit radio button -run_streamlit = tk.IntVar(value=0) +run_streamlit = tk.IntVar(value=1) callback_radio_st_run = radiobutton_event(run_streamlit) ctk_radio_st_autorun_1 = customtkinter.CTkRadioButton( app, From ea08a8a44b744503e1d4ad73062d102e2bd94a09 Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Fri, 28 Feb 2025 13:04:21 +0100 Subject: [PATCH 13/91] :art: format app script further --- gui/app.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/gui/app.py b/gui/app.py index 0b16eca..8c1a591 100644 --- a/gui/app.py +++ b/gui/app.py @@ -31,6 +31,7 @@ customtkinter.set_default_color_theme("dark-blue") +########################################################################################## # callbacks def create_run_vuegen(is_dir, config_path, report_type, run_streamlit): def inner(): @@ -63,12 +64,7 @@ def radio_button_callback(): return radio_button_callback -# Options - -# get list of report types from Enum -report_types = [report_type.value.lower() for report_type in ReportType] - - +########################################################################################## # APP app = customtkinter.CTk() app.geometry("460x400") @@ -110,6 +106,9 @@ def radio_button_callback(): ########################################################################################## # Report type dropdown +# - get list of report types from Enum +report_types = [report_type.value.lower() for report_type in ReportType] + ctk_label_report = customtkinter.CTkLabel( app, text="Select type of report to generate (use streamlit for now)", From c65ce378cf756fd34452afdbfad8b34b27e8c3ba Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Fri, 28 Feb 2025 13:34:31 +0100 Subject: [PATCH 14/91] =?UTF-8?q?=E2=8F=AA=20undo=20deletion=20of=20depend?= =?UTF-8?q?encies?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 0bcb820..e139c59 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,6 +55,7 @@ build-backend = "poetry_dynamic_versioning.backend" # https://stackoverflow.com/a/60990574/9684872 [tool.poetry.extras] +docs = ["sphinx", "sphinx-book-theme", "myst-nb", "ipywidgets", "sphinx-new-tab-link", "jupytext"] gui = ["customtkinter"] [tool.poetry.scripts] From f13f38340aae584012acb19ede9b1bc61f6b93ca Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Fri, 28 Feb 2025 13:49:48 +0100 Subject: [PATCH 15/91] :sparkles::art: Try to build executable on ubuntu (+ formating) --- .github/workflows/cdci.yml | 43 +++++++++++++++++++++++++++++++------- 1 file changed, 35 insertions(+), 8 deletions(-) diff --git a/.github/workflows/cdci.yml b/.github/workflows/cdci.yml index 1fa82e6..a2dc0ca 100644 --- a/.github/workflows/cdci.yml +++ b/.github/workflows/cdci.yml @@ -7,7 +7,7 @@ on: branches: [main] release: types: [published] - + jobs: test: name: Unittests+streamlit @@ -17,14 +17,14 @@ jobs: python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] steps: - uses: actions/checkout@v4 - - uses: psf/black@stable + - uses: psf/black@stable - uses: isort/isort-action@v1 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - cache: 'pip' # caching pip dependencies - cache-dependency-path: '**/pyproject.toml' + cache: "pip" # caching pip dependencies + cache-dependency-path: "**/pyproject.toml" - name: Install dependencies run: | python -m pip install --upgrade pip @@ -37,7 +37,7 @@ jobs: cd docs vuegen --directory example_data/Earth_microbiome_vuegen_demo_notebook vuegen --config example_data/Earth_microbiome_vuegen_demo_notebook/Earth_microbiome_vuegen_demo_notebook_config.yaml - + other-reports: name: Integration tests runs-on: ubuntu-latest @@ -50,8 +50,8 @@ jobs: uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - cache: 'pip' # caching pip dependencies - cache-dependency-path: '**/pyproject.toml' + cache: "pip" # caching pip dependencies + cache-dependency-path: "**/pyproject.toml" - name: Install dependencies run: | pip install --upgrade pip @@ -110,7 +110,7 @@ jobs: steps: - uses: actions/checkout@v4 - + - uses: actions/setup-python@v5 with: python-version: "3.11" @@ -120,3 +120,30 @@ jobs: run: python -m build - name: Publish package distributions to PyPI uses: pypa/gh-action-pypi-publish@release/v1 + + build-executable: + name: Build executable + runs-on: ${{ matrix.os }} + needs: + - test + - other-reports + strategy: + matrix: + python-version: ["3.11"] + os: ["ubuntu-latest"] #, "macos-13", "macos-latest", "windows-latest",] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Install PyInstaller + run: python -m pip install pyinstaller + - name: Build executable + run: | + cd gui + pyinstaller -n vuegen_gui --no-confirm --onedir --collect-all pyvis --collect-all streamlit --collect-all st_aggrid --collect-all customtkinter app.py + - name: Upload executable + uses: actions/upload-artifact@v4 + with: + name: vuegen_gui_${{ matrix.os }} + path: gui/dist/ From 71cc175e28e290239919214afd2ea96f268712cf Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Fri, 28 Feb 2025 14:02:30 +0100 Subject: [PATCH 16/91] =?UTF-8?q?:bug:=20cli=20different=20than=20on=20MAC?= =?UTF-8?q?,=20=E2=9A=A1=20speed=20up=20testing=20for=20now?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - only run againt Python 3.11 --- .github/workflows/cdci.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/cdci.yml b/.github/workflows/cdci.yml index a2dc0ca..b585ea6 100644 --- a/.github/workflows/cdci.yml +++ b/.github/workflows/cdci.yml @@ -14,7 +14,8 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] + # python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] + python-version: ["3.11"] steps: - uses: actions/checkout@v4 - uses: psf/black@stable @@ -43,7 +44,8 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] + # python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] + python-version: ["3.11"] steps: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} @@ -141,7 +143,7 @@ jobs: - name: Build executable run: | cd gui - pyinstaller -n vuegen_gui --no-confirm --onedir --collect-all pyvis --collect-all streamlit --collect-all st_aggrid --collect-all customtkinter app.py + pyinstaller -n vuegen_gui -D --collect-all pyvis --collect-all streamlit --collect-all st_aggrid --collect-all customtkinter app.py - name: Upload executable uses: actions/upload-artifact@v4 with: From 8a77e2f43529f081ac4ce57f9d4d33a7240a82bc Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Fri, 28 Feb 2025 14:12:58 +0100 Subject: [PATCH 17/91] :bug: install vuegen itself with gui deps --- .github/workflows/cdci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cdci.yml b/.github/workflows/cdci.yml index b585ea6..21374b1 100644 --- a/.github/workflows/cdci.yml +++ b/.github/workflows/cdci.yml @@ -138,8 +138,8 @@ jobs: - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - - name: Install PyInstaller - run: python -m pip install pyinstaller + - name: Install VueGen GUI and pyinstaller + run: python -m pip install '.[gui]' pyinstaller - name: Build executable run: | cd gui From 8712c2a0e3d219358bfe99ade991c9828c7ec808 Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Fri, 28 Feb 2025 14:31:11 +0100 Subject: [PATCH 18/91] :art: introduce labels for runner images to be able to specify what is what more easily --- .github/workflows/cdci.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/cdci.yml b/.github/workflows/cdci.yml index 21374b1..4989f5a 100644 --- a/.github/workflows/cdci.yml +++ b/.github/workflows/cdci.yml @@ -125,14 +125,18 @@ jobs: build-executable: name: Build executable - runs-on: ${{ matrix.os }} + runs-on: ${{ matrix.os.runner }} needs: - test - other-reports strategy: matrix: python-version: ["3.11"] - os: ["ubuntu-latest"] #, "macos-13", "macos-latest", "windows-latest",] + os: + # https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/running-variations-of-jobs-in-a-workflow#example-using-a-multi-dimension-matrix + - runner: "ubuntu-latest" + label: "ubuntu-latest_LTS-x64" + # "macos-13", "macos-latest", "windows-latest",] steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 @@ -147,5 +151,5 @@ jobs: - name: Upload executable uses: actions/upload-artifact@v4 with: - name: vuegen_gui_${{ matrix.os }} + name: vuegen_gui_${{ matrix.os.label }} path: gui/dist/ From e9c00473433374add1e5c1721eefb3da8d2a74be Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Fri, 28 Feb 2025 14:44:44 +0100 Subject: [PATCH 19/91] :sparkles: add other runner OS platforms with labels --- .github/workflows/cdci.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cdci.yml b/.github/workflows/cdci.yml index 4989f5a..5347f1b 100644 --- a/.github/workflows/cdci.yml +++ b/.github/workflows/cdci.yml @@ -124,7 +124,7 @@ jobs: uses: pypa/gh-action-pypi-publish@release/v1 build-executable: - name: Build executable + name: Build executable {{ matrix.os.label }} runs-on: ${{ matrix.os.runner }} needs: - test @@ -136,7 +136,12 @@ jobs: # https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/running-variations-of-jobs-in-a-workflow#example-using-a-multi-dimension-matrix - runner: "ubuntu-latest" label: "ubuntu-latest_LTS-x64" - # "macos-13", "macos-latest", "windows-latest",] + - runner: "macos-13" + label: "macos-x64" + - runner: "macos-latest" + label: "macos-arm64" + - runner: "windows-latest" + label: "windows-x64" steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 From 6a0fc786fe8f33127f84ef6acdce77323d93433e Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Fri, 28 Feb 2025 15:35:21 +0100 Subject: [PATCH 20/91] :bug: poetry optional dependencies need to be explicit, fix name of job - extras need to be specified as additinal dependencies - build name of job correctly --- .github/workflows/cdci.yml | 2 +- pyproject.toml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cdci.yml b/.github/workflows/cdci.yml index 5347f1b..7aeb653 100644 --- a/.github/workflows/cdci.yml +++ b/.github/workflows/cdci.yml @@ -124,7 +124,7 @@ jobs: uses: pypa/gh-action-pypi-publish@release/v1 build-executable: - name: Build executable {{ matrix.os.label }} + name: Build executable: vuegen_gui_${{ matrix.os.label }} runs-on: ${{ matrix.os.runner }} needs: - test diff --git a/pyproject.toml b/pyproject.toml index 98e9949..d4a8cb8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,6 +39,7 @@ myst-nb = {version="*", optional=true} ipywidgets = {version="*", optional=true} sphinx-new-tab-link = {version = "!=0.2.2", optional=true} jupytext = {version="*", optional=true} +customtkinter = {version="*", optional=true} [tool.poetry.group.dev.dependencies] ipykernel = {version="^6.29.5", optional=true} From ef61a0371e3ca2c1c0cf5ee2caf3ddf94038db4a Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Fri, 28 Feb 2025 15:42:02 +0100 Subject: [PATCH 21/91] :bug: colon not allowed in name... --- .github/workflows/cdci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cdci.yml b/.github/workflows/cdci.yml index 7aeb653..491caf6 100644 --- a/.github/workflows/cdci.yml +++ b/.github/workflows/cdci.yml @@ -124,7 +124,7 @@ jobs: uses: pypa/gh-action-pypi-publish@release/v1 build-executable: - name: Build executable: vuegen_gui_${{ matrix.os.label }} + name: Build executable - vuegen_gui_${{ matrix.os.label }} runs-on: ${{ matrix.os.runner }} needs: - test @@ -148,7 +148,7 @@ jobs: with: python-version: ${{ matrix.python-version }} - name: Install VueGen GUI and pyinstaller - run: python -m pip install '.[gui]' pyinstaller + run: python -m pip install ".[gui]" pyinstaller - name: Build executable run: | cd gui From c55208a623e488408a7a74d63c0e37b1519e7121 Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Fri, 28 Feb 2025 16:32:54 +0100 Subject: [PATCH 22/91] :rewind: undo command change --- src/vuegen/quarto_reportview.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/vuegen/quarto_reportview.py b/src/vuegen/quarto_reportview.py index 68725fb..4f4d695 100644 --- a/src/vuegen/quarto_reportview.py +++ b/src/vuegen/quarto_reportview.py @@ -166,7 +166,7 @@ def run_report(self, output_dir: str = BASE_DIR) -> None: subprocess.run( [ "quarto", - "render", + "convert", os.path.join(output_dir, f"{self.BASE_DIR}.qmd"), ], check=True, From b9a66ffbfeb7ea55c306671a898dfc4d1e4ae760 Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Fri, 28 Feb 2025 16:44:15 +0100 Subject: [PATCH 23/91] :sparkles: define basic example as default, ship it with bundle Always have something to run. --- .github/workflows/cdci.yml | 4 ++-- gui/README.md | 3 ++- gui/app.py | 23 ++++++++++++++++++++++- 3 files changed, 26 insertions(+), 4 deletions(-) diff --git a/.github/workflows/cdci.yml b/.github/workflows/cdci.yml index 491caf6..cebbbb6 100644 --- a/.github/workflows/cdci.yml +++ b/.github/workflows/cdci.yml @@ -124,7 +124,7 @@ jobs: uses: pypa/gh-action-pypi-publish@release/v1 build-executable: - name: Build executable - vuegen_gui_${{ matrix.os.label }} + name: Build-exe-vuegen_gui_${{ matrix.os.label }} runs-on: ${{ matrix.os.runner }} needs: - test @@ -152,7 +152,7 @@ jobs: - name: Build executable run: | cd gui - pyinstaller -n vuegen_gui -D --collect-all pyvis --collect-all streamlit --collect-all st_aggrid --collect-all customtkinter app.py + pyinstaller -n vuegen_gui -D --collect-all pyvis --collect-all streamlit --collect-all st_aggrid --collect-all customtkinter --add-data ../docs/example_data/Basic_example_vuegen_demo_notebook:example_data/Basic_example_vuegen_demo_notebook app.py - name: Upload executable uses: actions/upload-artifact@v4 with: diff --git a/gui/README.md b/gui/README.md index c71974e..b59c12b 100644 --- a/gui/README.md +++ b/gui/README.md @@ -29,11 +29,12 @@ as the streamlit server and the GUI itself. # from this README folder pyinstaller \ -n vuegen_gui \ ---no-confirm \ +--noconfirm \ --onedir \ --collect-all pyvis \ --collect-all streamlit \ --collect-all st_aggrid \ --collect-all customtkinter \ +--add-data ../docs/example_data/Basic_example_vuegen_demo_notebook:example_data/Basic_example_vuegen_demo_notebook \ app.py ``` diff --git a/gui/app.py b/gui/app.py index 8c1a591..67ce743 100644 --- a/gui/app.py +++ b/gui/app.py @@ -21,6 +21,7 @@ import sys import tkinter as tk +from pathlib import Path import customtkinter @@ -30,6 +31,26 @@ customtkinter.set_appearance_mode("system") customtkinter.set_default_color_theme("dark-blue") +app_path = Path(__file__).absolute() +print("app_path:", app_path) + +if getattr(sys, "frozen", False) and hasattr(sys, "_MEIPASS"): + # PyInstaller bundeled case + path_to_dat = ( + app_path.parent / "example_data/Basic_example_vuegen_demo_notebook" + ).resolve() +elif app_path.parent.name == "gui": + # should be always the case for GUI run from command line + path_to_dat = ( + app_path.parent + / ".." + / "docs" + / "example_data" + / "Basic_example_vuegen_demo_notebook" + ).resolve() +else: + path_to_dat = "docs/example_data/Basic_example_vuegen_demo_notebook" + ########################################################################################## # callbacks @@ -96,7 +117,7 @@ def radio_button_callback(): ) ctk_radio_config_1.grid(row=1, column=1, padx=20, pady=2) -config_path = tk.StringVar() +config_path = tk.StringVar(value=str(path_to_dat)) config_path_entry = customtkinter.CTkEntry( app, width=400, From a53e49d0ec628b3fcafbcd70786ce2912f186b47 Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Mon, 3 Mar 2025 10:03:13 +0100 Subject: [PATCH 24/91] :art: make process label somehow fit UI limitations - for GitHub Actions View in the browser --- .github/workflows/cdci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cdci.yml b/.github/workflows/cdci.yml index cebbbb6..d8891fa 100644 --- a/.github/workflows/cdci.yml +++ b/.github/workflows/cdci.yml @@ -124,7 +124,7 @@ jobs: uses: pypa/gh-action-pypi-publish@release/v1 build-executable: - name: Build-exe-vuegen_gui_${{ matrix.os.label }} + name: Build-exe-${{ matrix.os.label }} runs-on: ${{ matrix.os.runner }} needs: - test From 0009bb8a0f7f4842abf2641f35c793e969b247b4 Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Mon, 3 Mar 2025 10:35:11 +0100 Subject: [PATCH 25/91] :bug: acutally get variable value, not test instance exist - before the existence was just tested, leading to always True (?) --- gui/app.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gui/app.py b/gui/app.py index 67ce743..74f730d 100644 --- a/gui/app.py +++ b/gui/app.py @@ -57,14 +57,14 @@ def create_run_vuegen(is_dir, config_path, report_type, run_streamlit): def inner(): args = ["vuegen"] - if is_dir: + if is_dir.get(): args.append("--directory") else: args.append("--config") args.append(config_path.get()) args.append("--report_type") args.append(report_type.get()) - if run_streamlit: + if run_streamlit.get(): args.append("--streamlit_autorun") print("args:", args) sys.argv = args From 29d1d415422ca09b547e92d1a15803d25b27bc09 Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Mon, 3 Mar 2025 10:22:03 +0000 Subject: [PATCH 26/91] :rewind: restore main branch commands (render, not convert) --- src/vuegen/quarto_reportview.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/vuegen/quarto_reportview.py b/src/vuegen/quarto_reportview.py index 4f4d695..68725fb 100644 --- a/src/vuegen/quarto_reportview.py +++ b/src/vuegen/quarto_reportview.py @@ -166,7 +166,7 @@ def run_report(self, output_dir: str = BASE_DIR) -> None: subprocess.run( [ "quarto", - "convert", + "render", os.path.join(output_dir, f"{self.BASE_DIR}.qmd"), ], check=True, From d5e344d8a65a243c23964186b8498bab342a9176 Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Mon, 3 Mar 2025 13:24:40 +0100 Subject: [PATCH 27/91] :truck::fire: remove cli exectuable for GUI bundle only MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 🚚 move some hints to GUI README --- executables/README.md | 39 ----------------------------- executables/vuegen.spec | 55 ----------------------------------------- gui/README.md | 22 +++++++++++++++++ 3 files changed, 22 insertions(+), 94 deletions(-) delete mode 100644 executables/README.md delete mode 100644 executables/vuegen.spec diff --git a/executables/README.md b/executables/README.md deleted file mode 100644 index 385c57f..0000000 --- a/executables/README.md +++ /dev/null @@ -1,39 +0,0 @@ -# Pyinstaller one folder executable - -- pyvis templates were not copied, so make these explicit (see [this](https://stackoverflow.com/a/72687433/9684872)) - -```bash -# from this README folder -pyinstaller -D --collect-all pyvis --collect-all streamlit --collect-all st_aggrid -n vuegen ../src/vuegen/__main__.py -``` - -```bash -# other pyinstaller options ---noconfirm # is used to avoid the prompt for overwriting the existing dist folder -``` - -## Pyinstaller options - -```bash -What to generate: - -D, --onedir Create a one-folder bundle containing an executable (default) - -F, --onefile Create a one-file bundled executable. - --specpath DIR Folder to store the generated spec file (default: current directory) - -n NAME, --name NAME Name to assign to the bundled app and spec file (default: first script's basename) -Windows and macOS specific options: - -c, --console, --nowindowed - Open a console window for standard i/o (default). On Windows this option has no effect if the first script is a - '.pyw' file. - -w, --windowed, --noconsole - Windows and macOS: do not provide a console window for standard i/o. On macOS this also triggers building a - macOS .app bundle. On Windows this option is automatically set if the first script is a '.pyw' file. This option - is ignored on *NIX systems. -``` - -## Using bundled executable - -try using basic example - -```bash -./dist/vuegen/vuegen -d ../docs/example_data/Basic_example_vuegen_demo_notebook -st_autorun -``` diff --git a/executables/vuegen.spec b/executables/vuegen.spec deleted file mode 100644 index 463d64d..0000000 --- a/executables/vuegen.spec +++ /dev/null @@ -1,55 +0,0 @@ -# -*- mode: python ; coding: utf-8 -*- -from PyInstaller.utils.hooks import collect_all - -datas = [] -binaries = [] -hiddenimports = [] -tmp_ret = collect_all('pyvis') -datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2] -tmp_ret = collect_all('streamlit') -datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2] -tmp_ret = collect_all('st_aggrid') -datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2] - - -a = Analysis( - ['../src/vuegen/__main__.py'], - pathex=[], - binaries=binaries, - datas=datas, - hiddenimports=hiddenimports, - hookspath=[], - hooksconfig={}, - runtime_hooks=[], - excludes=[], - noarchive=False, - optimize=0, -) -pyz = PYZ(a.pure) - -exe = EXE( - pyz, - a.scripts, - [], - exclude_binaries=True, - name='vuegen', - debug=False, - bootloader_ignore_signals=False, - strip=False, - upx=True, - console=True, - disable_windowed_traceback=False, - argv_emulation=False, - target_arch=None, - codesign_identity=None, - entitlements_file=None, -) -coll = COLLECT( - exe, - a.binaries, - a.datas, - strip=False, - upx=True, - upx_exclude=[], - name='vuegen', -) diff --git a/gui/README.md b/gui/README.md index b59c12b..68ce6c3 100644 --- a/gui/README.md +++ b/gui/README.md @@ -38,3 +38,25 @@ pyinstaller \ --add-data ../docs/example_data/Basic_example_vuegen_demo_notebook:example_data/Basic_example_vuegen_demo_notebook \ app.py ``` + +- pyvis templates were not copied, so make these explicit (see [this](https://stackoverflow.com/a/72687433/9684872)) +- same for streamlit, customtkinter and st_aggrid +- might be copying too much, but for now we go the safe route + +## relevant Pyinstaller options + +```bash +What to generate: + -D, --onedir Create a one-folder bundle containing an executable (default) + -F, --onefile Create a one-file bundled executable. + --specpath DIR Folder to store the generated spec file (default: current directory) + -n NAME, --name NAME Name to assign to the bundled app and spec file (default: first script's basename) +Windows and macOS specific options: + -c, --console, --nowindowed + Open a console window for standard i/o (default). On Windows this option has no effect if the first script is a + '.pyw' file. + -w, --windowed, --noconsole + Windows and macOS: do not provide a console window for standard i/o. On macOS this also triggers building a + macOS .app bundle. On Windows this option is automatically set if the first script is a '.pyw' file. This option + is ignored on *NIX systems. +``` From 644435359b5f6e28811c33ae36e1e9c58519efa0 Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Mon, 3 Mar 2025 13:39:34 +0100 Subject: [PATCH 28/91] =?UTF-8?q?=F0=9F=94=A7=20Only=20show=20streamlit=20?= =?UTF-8?q?for=20now=20(quarto=20not=20yet=20bundled)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- gui/app.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gui/app.py b/gui/app.py index 74f730d..bc1f7de 100644 --- a/gui/app.py +++ b/gui/app.py @@ -129,10 +129,10 @@ def radio_button_callback(): # Report type dropdown # - get list of report types from Enum report_types = [report_type.value.lower() for report_type in ReportType] - +report_types = ["streamlit"] # ! for now, only streamlit is supported ctk_label_report = customtkinter.CTkLabel( app, - text="Select type of report to generate (use streamlit for now)", + text="Select type of report to generate (using only streamlit for now)", ) ctk_label_report.grid(row=3, column=0, columnspan=2, padx=20, pady=20) report_type = tk.StringVar(value=report_types[0]) From 1e236f8dd6c48df4c44687cd1aed79037ee24ed7 Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Mon, 3 Mar 2025 19:42:46 +0100 Subject: [PATCH 29/91] :construction: add quarto execution for bundle - execution still fails - convert and render without execution? - add all report types to GUI (for testing) - do not stop app when quarto is not in PATH --- gui/app.py | 3 +- src/vuegen/quarto_reportview.py | 76 +++++++++++++++++++++------------ src/vuegen/report_generator.py | 5 ++- 3 files changed, 55 insertions(+), 29 deletions(-) diff --git a/gui/app.py b/gui/app.py index bc1f7de..20ac390 100644 --- a/gui/app.py +++ b/gui/app.py @@ -34,6 +34,8 @@ app_path = Path(__file__).absolute() print("app_path:", app_path) +########################################################################################## +# Path to example data dependend on how the GUI is run if getattr(sys, "frozen", False) and hasattr(sys, "_MEIPASS"): # PyInstaller bundeled case path_to_dat = ( @@ -129,7 +131,6 @@ def radio_button_callback(): # Report type dropdown # - get list of report types from Enum report_types = [report_type.value.lower() for report_type in ReportType] -report_types = ["streamlit"] # ! for now, only streamlit is supported ctk_label_report = customtkinter.CTkLabel( app, text="Select type of report to generate (using only streamlit for now)", diff --git a/src/vuegen/quarto_reportview.py b/src/vuegen/quarto_reportview.py index 68725fb..31b63b6 100644 --- a/src/vuegen/quarto_reportview.py +++ b/src/vuegen/quarto_reportview.py @@ -25,6 +25,7 @@ def __init__(self, report: r.Report, report_type: r.ReportType): if getattr(sys, "frozen", False) and hasattr(sys, "_MEIPASS"): self.report.logger.info("running in a PyInstaller bundle") self.BUNDLED_EXECUTION = True + self.report.logger.debug(f"sys._MEIPASS: {sys._MEIPASS}") else: self.report.logger.info("running in a normal Python process") @@ -161,43 +162,64 @@ def run_report(self, output_dir: str = BASE_DIR) -> None: output_dir : str, optional The folder where the report was generated (default is 'sections'). """ - try: - if not self.BUNDLED_EXECUTION: - subprocess.run( - [ - "quarto", - "render", - os.path.join(output_dir, f"{self.BASE_DIR}.qmd"), - ], - check=True, - ) + # from quarto_cli import run_quarto # entrypoint of quarto-cli not in module? + + file_path_to_qmd = os.path.join(output_dir, f"{self.BASE_DIR}.qmd") + args = ["quarto", "render", file_path_to_qmd] + self.report.logger.info( + f"Running '{self.report.title}' '{self.report_type}' report with {args!r}" + ) + if not self.BUNDLED_EXECUTION: + subprocess.run( + args, + check=True, + ) + try: if self.report_type == r.ReportType.JUPYTER: + args = ["quarto", "convert", file_path_to_qmd] subprocess.run( - [ - "quarto", - "convert", - os.path.join(output_dir, f"{self.BASE_DIR}.qmd"), - ], + args, check=True, ) + self.report.logger.info( + f"Converted '{self.report.title}' '{self.report_type}' report to Jupyter Notebook after execution" + ) self.report.logger.info( f"'{self.report.title}' '{self.report_type}' report rendered" ) - else: - self.report.logger.info( - f"Quarto report '{self.report.title}' '{self.report_type}' cannot " - "be run in a PyInstaller bundle" + except subprocess.CalledProcessError as e: + self.report.logger.error( + f"Error running '{self.report.title}' {self.report_type} report: {str(e)}" ) - except subprocess.CalledProcessError as e: - self.report.logger.error( - f"Error running '{self.report.title}' {self.report_type} report: {str(e)}" + raise + except FileNotFoundError as e: + self.report.logger.error( + f"Quarto is not installed. Please install Quarto to run the report: {str(e)}" + ) + raise + else: + quarto_path = Path(sys._MEIPASS) / "quarto_cli" / "bin" / "quarto" + if sys.platform == "win32": + # ! to check + quarto_path = self.quarto_path.with_suffix(".exe") + self.report.logger.info(f"quarto_path: {quarto_path}") + + args = [f"{quarto_path}", "convert", file_path_to_qmd] + subprocess.run( + args, + check=True, ) - raise - except FileNotFoundError as e: - self.report.logger.error( - f"Quarto is not installed. Please install Quarto to run the report: {str(e)}" + self.report.logger.info( + f"Converted '{self.report.title}' '{self.report_type}' report to Jupyter Notebook after execution" + ) + notebook_filename = Path(file_path_to_qmd).with_suffix(".ipynb") + # ipynb will not be executed per default + # ! check if execution works although qmd cannot be executed... + args = [f"{quarto_path}", "render", notebook_filename] + subprocess.run( + args, + check=True, ) - raise def _create_yaml_header(self) -> str: """ diff --git a/src/vuegen/report_generator.py b/src/vuegen/report_generator.py index d5b656c..e09900d 100644 --- a/src/vuegen/report_generator.py +++ b/src/vuegen/report_generator.py @@ -1,5 +1,6 @@ import logging import shutil +import sys from .config_manager import ConfigManager from .quarto_reportview import QuartoReportView @@ -67,7 +68,9 @@ def get_report( else: # Check if Quarto is installed - if shutil.which("quarto") is None: + if shutil.which("quarto") is None and not hasattr( + sys, "_MEIPASS" + ): # ? and not getattr(sys, "frozen", False) logger.error( "Quarto is not installed. Please install Quarto before generating this report type." ) From a13e784494b28635c079099c709aa73d8989a43b Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Wed, 5 Mar 2025 11:52:30 +0100 Subject: [PATCH 30/91] :sparkles: get quarto html output from bundle - use quarto (pandoc?) to convert qmd to ipynb - use nbconvert and a copied Python executable to execute notebook - use quarto (pandoc?) ot convert executed ipynb to desired format --- .github/workflows/cdci.yml | 1 + gui/README.md | 11 +++++++++++ gui/copy_python_executable.py | 6 ++++++ src/vuegen/quarto_reportview.py | 22 +++++++++++++++++----- 4 files changed, 35 insertions(+), 5 deletions(-) create mode 100644 gui/copy_python_executable.py diff --git a/.github/workflows/cdci.yml b/.github/workflows/cdci.yml index d8891fa..46a3b4e 100644 --- a/.github/workflows/cdci.yml +++ b/.github/workflows/cdci.yml @@ -153,6 +153,7 @@ jobs: run: | cd gui pyinstaller -n vuegen_gui -D --collect-all pyvis --collect-all streamlit --collect-all st_aggrid --collect-all customtkinter --add-data ../docs/example_data/Basic_example_vuegen_demo_notebook:example_data/Basic_example_vuegen_demo_notebook app.py + python copy_python_executable.py - name: Upload executable uses: actions/upload-artifact@v4 with: diff --git a/gui/README.md b/gui/README.md index 68ce6c3..923f5d6 100644 --- a/gui/README.md +++ b/gui/README.md @@ -60,3 +60,14 @@ Windows and macOS specific options: macOS .app bundle. On Windows this option is automatically set if the first script is a '.pyw' file. This option is ignored on *NIX systems. ``` + +## Quarto notebook execution + +- add python exe to bundle as suggested [on stackoverflow](https://stackoverflow.com/a/72639099/9684872) +- use [copy_python_executable.py](copy_python_executable.py) to copy the python executable to the bundle after PyInstaller is done + +Basic workflow for bundle: + +1. use quarto (pandoc?) to convert qmd to ipynb +1. use nbconvert and a copied Python executable to execute notebook +1. use quarto (pandoc?) ot convert executed ipynb to desired format diff --git a/gui/copy_python_executable.py b/gui/copy_python_executable.py new file mode 100644 index 0000000..710aa4c --- /dev/null +++ b/gui/copy_python_executable.py @@ -0,0 +1,6 @@ +import shutil +import sys +from pathlib import Path + +python_exe = Path(sys.executable).with_stem("python") +shutil.copy(python_exe, "dist/vuegen_gui/_internal") diff --git a/src/vuegen/quarto_reportview.py b/src/vuegen/quarto_reportview.py index 31b63b6..93180ce 100644 --- a/src/vuegen/quarto_reportview.py +++ b/src/vuegen/quarto_reportview.py @@ -199,9 +199,9 @@ def run_report(self, output_dir: str = BASE_DIR) -> None: raise else: quarto_path = Path(sys._MEIPASS) / "quarto_cli" / "bin" / "quarto" - if sys.platform == "win32": - # ! to check - quarto_path = self.quarto_path.with_suffix(".exe") + _sys_exe = sys.executable + # set executable to the bundled python (manually added to bundle) + sys.executable = str(Path(sys._MEIPASS) / "python") self.report.logger.info(f"quarto_path: {quarto_path}") args = [f"{quarto_path}", "convert", file_path_to_qmd] @@ -213,13 +213,25 @@ def run_report(self, output_dir: str = BASE_DIR) -> None: f"Converted '{self.report.title}' '{self.report_type}' report to Jupyter Notebook after execution" ) notebook_filename = Path(file_path_to_qmd).with_suffix(".ipynb") - # ipynb will not be executed per default - # ! check if execution works although qmd cannot be executed... + # quarto does not try to execute ipynb files, just render them + # execute manually using bundled python binary + # https://nbconvert.readthedocs.io/en/latest/execute_api.html + import nbformat + from nbconvert.preprocessors import ExecutePreprocessor + + with open(notebook_filename, encoding="utf-8") as f: + nb = nbformat.read(f, as_version=4) + ep = ExecutePreprocessor(timeout=600, kernel_name="python3") + nb, _ = ep.preprocess(nb, {"metadata": {"path": "./"}}) + with open(notebook_filename, "w", encoding="utf-8") as f: + nbformat.write(nb, f) + # quarto does not try execute ipynb files per default, just render these args = [f"{quarto_path}", "render", notebook_filename] subprocess.run( args, check=True, ) + sys.executable = _sys_exe def _create_yaml_header(self) -> str: """ From d46bc02444e4bf8d541ede87ad3dc31d40372527 Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Wed, 5 Mar 2025 14:11:50 +0100 Subject: [PATCH 31/91] :bug: add bundle dependencies, RadioButton toggle - radiobutton values are not correctly used in bundled app when built in runner - works locally (woth local bundle, and direct execution of app) --- .github/workflows/cdci.yml | 2 +- gui/app.py | 22 ++++++++++++---------- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/.github/workflows/cdci.yml b/.github/workflows/cdci.yml index 46a3b4e..d5a964b 100644 --- a/.github/workflows/cdci.yml +++ b/.github/workflows/cdci.yml @@ -152,7 +152,7 @@ jobs: - name: Build executable run: | cd gui - pyinstaller -n vuegen_gui -D --collect-all pyvis --collect-all streamlit --collect-all st_aggrid --collect-all customtkinter --add-data ../docs/example_data/Basic_example_vuegen_demo_notebook:example_data/Basic_example_vuegen_demo_notebook app.py + pyinstaller -n vuegen_gui -D --collect-all pyvis --collect-all streamlit --collect-all st_aggrid --collect-all customtkinter --collect-all quarto_cli --collect-all jupyter_core --collect-all yaml --collect-all ipykernel --collect-all nbconvert --collect-all notebook --collect-all ipywidgets --collect-all jupyter_console --collect-all jupyter_client -add-data ../docs/example_data/Basic_example_vuegen_demo_notebook:example_data/Basic_example_vuegen_demo_notebook app.py python copy_python_executable.py - name: Upload executable uses: actions/upload-artifact@v4 diff --git a/gui/app.py b/gui/app.py index 20ac390..7bd7ab5 100644 --- a/gui/app.py +++ b/gui/app.py @@ -59,6 +59,7 @@ def create_run_vuegen(is_dir, config_path, report_type, run_streamlit): def inner(): args = ["vuegen"] + print(f"{is_dir.get() = }") if is_dir.get(): args.append("--directory") else: @@ -66,6 +67,7 @@ def inner(): args.append(config_path.get()) args.append("--report_type") args.append(report_type.get()) + print(f"{run_streamlit.get() = }") if run_streamlit.get(): args.append("--streamlit_autorun") print("args:", args) @@ -80,9 +82,9 @@ def optionmenu_callback(choice): print("optionmenu dropdown clicked:", choice) -def radiobutton_event(value): +def radiobutton_event(value, name="radiobutton"): def radio_button_callback(): - print("radiobutton toggled, current value:", value.get()) + print(f"{name} toggled, current value:", value.get()) return radio_button_callback @@ -100,14 +102,14 @@ def radio_button_callback(): text="Add path to config file or directory. Select radio button accordingly", ) ctk_label_config.grid(row=0, column=0, columnspan=2, padx=20, pady=20) -is_dir = tk.IntVar(value=1) -callback_radio_config = radiobutton_event(is_dir) +is_dir = tk.BooleanVar(value=True) +callback_radio_config = radiobutton_event(is_dir, name="is_dir") ctk_radio_config_0 = customtkinter.CTkRadioButton( app, text="Use config", command=callback_radio_config, variable=is_dir, - value=0, + value=False, ) ctk_radio_config_0.grid(row=1, column=0, padx=20, pady=2) ctk_radio_config_1 = customtkinter.CTkRadioButton( @@ -115,7 +117,7 @@ def radio_button_callback(): text="Use dir", command=callback_radio_config, variable=is_dir, - value=1, + value=True, ) ctk_radio_config_1.grid(row=1, column=1, padx=20, pady=2) @@ -150,12 +152,12 @@ def radio_button_callback(): ########################################################################################## # Run Streamlit radio button -run_streamlit = tk.IntVar(value=1) -callback_radio_st_run = radiobutton_event(run_streamlit) +run_streamlit = tk.BooleanVar(value=True) +callback_radio_st_run = radiobutton_event(run_streamlit, name="run_streamlit") ctk_radio_st_autorun_1 = customtkinter.CTkRadioButton( app, text="autorun streamlit", - value=1, + value=True, variable=run_streamlit, command=callback_radio_st_run, ) @@ -163,7 +165,7 @@ def radio_button_callback(): ctk_radio_st_autorun_0 = customtkinter.CTkRadioButton( app, text="skip starting streamlit", - value=0, + value=False, variable=run_streamlit, command=callback_radio_st_run, ) From fb1f4a2b0a834de58d6ec2ae65e8df8f10f66a8d Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Wed, 5 Mar 2025 14:24:42 +0100 Subject: [PATCH 32/91] :bug: fix command om cdci --- .github/workflows/cdci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cdci.yml b/.github/workflows/cdci.yml index d5a964b..fb8173e 100644 --- a/.github/workflows/cdci.yml +++ b/.github/workflows/cdci.yml @@ -152,7 +152,7 @@ jobs: - name: Build executable run: | cd gui - pyinstaller -n vuegen_gui -D --collect-all pyvis --collect-all streamlit --collect-all st_aggrid --collect-all customtkinter --collect-all quarto_cli --collect-all jupyter_core --collect-all yaml --collect-all ipykernel --collect-all nbconvert --collect-all notebook --collect-all ipywidgets --collect-all jupyter_console --collect-all jupyter_client -add-data ../docs/example_data/Basic_example_vuegen_demo_notebook:example_data/Basic_example_vuegen_demo_notebook app.py + pyinstaller -n vuegen_gui -D --collect-all pyvis --collect-all streamlit --collect-all st_aggrid --collect-all customtkinter --collect-all quarto_cli --collect-all jupyter_core --collect-all yaml --collect-all ipykernel --collect-all nbconvert --collect-all notebook --collect-all ipywidgets --collect-all jupyter_console --collect-all jupyter_client --add-data ../docs/example_data/Basic_example_vuegen_demo_notebook:example_data/Basic_example_vuegen_demo_notebook app.py python copy_python_executable.py - name: Upload executable uses: actions/upload-artifact@v4 From d090c80412610543a8d82229cb4083626760103e Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Wed, 5 Mar 2025 14:34:14 +0100 Subject: [PATCH 33/91] :arg: reduce console output (global logging level is DEBUG?) --- src/vuegen/quarto_reportview.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/vuegen/quarto_reportview.py b/src/vuegen/quarto_reportview.py index 2c5dfa4..471bb11 100644 --- a/src/vuegen/quarto_reportview.py +++ b/src/vuegen/quarto_reportview.py @@ -1,3 +1,4 @@ +import logging import os import subprocess import sys @@ -219,6 +220,7 @@ def run_report(self, output_dir: str = BASE_DIR) -> None: with open(notebook_filename, encoding="utf-8") as f: nb = nbformat.read(f, as_version=4) + logging.getLogger("traitlets").setLevel(logging.INFO) ep = ExecutePreprocessor(timeout=600, kernel_name="python3") nb, _ = ep.preprocess(nb, {"metadata": {"path": "./"}}) with open(notebook_filename, "w", encoding="utf-8") as f: From e1150c0f26076b8e70248b63d52b18f1b71ab40f Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Wed, 5 Mar 2025 15:42:58 +0100 Subject: [PATCH 34/91] :bug: use macos-15 explicitly --- .github/workflows/cdci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/cdci.yml b/.github/workflows/cdci.yml index fb8173e..60a7870 100644 --- a/.github/workflows/cdci.yml +++ b/.github/workflows/cdci.yml @@ -137,9 +137,9 @@ jobs: - runner: "ubuntu-latest" label: "ubuntu-latest_LTS-x64" - runner: "macos-13" - label: "macos-x64" - - runner: "macos-latest" - label: "macos-arm64" + label: "macos-13-x64" + - runner: "macos-15" + label: "macos-15-arm64" - runner: "windows-latest" label: "windows-x64" steps: From 86c566057f9af0c031f6c02c26ca23fd7eaf6822 Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Wed, 5 Mar 2025 15:44:46 +0100 Subject: [PATCH 35/91] :art: format and rename callback factory --- gui/app.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/gui/app.py b/gui/app.py index 7bd7ab5..3f5d8a4 100644 --- a/gui/app.py +++ b/gui/app.py @@ -82,7 +82,7 @@ def optionmenu_callback(choice): print("optionmenu dropdown clicked:", choice) -def radiobutton_event(value, name="radiobutton"): +def create_radio_button_callback(value, name="radiobutton"): def radio_button_callback(): print(f"{name} toggled, current value:", value.get()) @@ -103,7 +103,7 @@ def radio_button_callback(): ) ctk_label_config.grid(row=0, column=0, columnspan=2, padx=20, pady=20) is_dir = tk.BooleanVar(value=True) -callback_radio_config = radiobutton_event(is_dir, name="is_dir") +callback_radio_config = create_radio_button_callback(is_dir, name="is_dir") ctk_radio_config_0 = customtkinter.CTkRadioButton( app, text="Use config", @@ -147,13 +147,12 @@ def radio_button_callback(): ) report_dropdown.grid(row=4, column=0, columnspan=2, padx=20, pady=20) -_report_type = report_dropdown.get() -print("report_type value:", _report_type) - ########################################################################################## # Run Streamlit radio button run_streamlit = tk.BooleanVar(value=True) -callback_radio_st_run = radiobutton_event(run_streamlit, name="run_streamlit") +callback_radio_st_run = create_radio_button_callback( + run_streamlit, name="run_streamlit" +) ctk_radio_st_autorun_1 = customtkinter.CTkRadioButton( app, text="autorun streamlit", From f9b79794c60b475b8934a6cd693019932a752aa0 Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Wed, 5 Mar 2025 15:45:28 +0100 Subject: [PATCH 36/91] :memo: add hint to execution procedure --- gui/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gui/README.md b/gui/README.md index 923f5d6..d9acc47 100644 --- a/gui/README.md +++ b/gui/README.md @@ -69,5 +69,6 @@ Windows and macOS specific options: Basic workflow for bundle: 1. use quarto (pandoc?) to convert qmd to ipynb -1. use nbconvert and a copied Python executable to execute notebook +1. use nbconvert programmatically (see [docs](https://nbconvert.readthedocs.io/en/latest/execute_api.html#example)) + and the bundled Python executable to execute notebook 1. use quarto (pandoc?) ot convert executed ipynb to desired format From 672bb711fc73309c380d24b232b5f00eb0af507b Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Wed, 5 Mar 2025 22:15:15 +0100 Subject: [PATCH 37/91] :bug: try to fix python executable path - just copy what ever is pointing to sys.executable - move to binaries folder Locally this all works, but it might be that the lcoal Python exe is somehow 'valid' (signed, etc) --- .github/workflows/cdci.yml | 2 +- gui/copy_python_executable.py | 6 ++++-- src/vuegen/quarto_reportview.py | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/cdci.yml b/.github/workflows/cdci.yml index 60a7870..f860d9c 100644 --- a/.github/workflows/cdci.yml +++ b/.github/workflows/cdci.yml @@ -152,7 +152,7 @@ jobs: - name: Build executable run: | cd gui - pyinstaller -n vuegen_gui -D --collect-all pyvis --collect-all streamlit --collect-all st_aggrid --collect-all customtkinter --collect-all quarto_cli --collect-all jupyter_core --collect-all yaml --collect-all ipykernel --collect-all nbconvert --collect-all notebook --collect-all ipywidgets --collect-all jupyter_console --collect-all jupyter_client --add-data ../docs/example_data/Basic_example_vuegen_demo_notebook:example_data/Basic_example_vuegen_demo_notebook app.py + pyinstaller -n vuegen_gui -D --collect-all pyvis --collect-all streamlit --collect-all st_aggrid --collect-all customtkinter --collect-all quarto_cli --collect-all jupyter_core --collect-all yaml --collect-all ipykernel --collect-all nbconvert --collect-all notebook --collect-all ipywidgets --collect-all jupyter_console --collect-all jupyter_client --add-data ../docs/example_data/Basic_example_vuegen_demo_notebook:example_data/Basic_example_vuegen_demo_notebook app.py python copy_python_executable.py - name: Upload executable uses: actions/upload-artifact@v4 diff --git a/gui/copy_python_executable.py b/gui/copy_python_executable.py index 710aa4c..3ba99ce 100644 --- a/gui/copy_python_executable.py +++ b/gui/copy_python_executable.py @@ -2,5 +2,7 @@ import sys from pathlib import Path -python_exe = Path(sys.executable).with_stem("python") -shutil.copy(python_exe, "dist/vuegen_gui/_internal") +python_exe = Path(sys.executable) #.with_stem("python") +print("Copying python executable:", python_exe) +shutil.copy(python_exe, "dist/vuegen_gui/_internal/python") +print("Done.") diff --git a/src/vuegen/quarto_reportview.py b/src/vuegen/quarto_reportview.py index 471bb11..1286a0b 100644 --- a/src/vuegen/quarto_reportview.py +++ b/src/vuegen/quarto_reportview.py @@ -200,7 +200,7 @@ def run_report(self, output_dir: str = BASE_DIR) -> None: quarto_path = Path(sys._MEIPASS) / "quarto_cli" / "bin" / "quarto" _sys_exe = sys.executable # set executable to the bundled python (manually added to bundle) - sys.executable = str(Path(sys._MEIPASS) / "python") + sys.executable = str(Path(sys._MEIPASS).parent / "python") self.report.logger.info(f"quarto_path: {quarto_path}") args = [f"{quarto_path}", "convert", file_path_to_qmd] From a3b4688bbf40e5a349f1a80396410ac18b61b6ec Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Wed, 5 Mar 2025 22:16:50 +0100 Subject: [PATCH 38/91] :art: format helper script --- gui/copy_python_executable.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gui/copy_python_executable.py b/gui/copy_python_executable.py index 3ba99ce..740bf6e 100644 --- a/gui/copy_python_executable.py +++ b/gui/copy_python_executable.py @@ -2,7 +2,7 @@ import sys from pathlib import Path -python_exe = Path(sys.executable) #.with_stem("python") +python_exe = Path(sys.executable) # .with_stem("python") print("Copying python executable:", python_exe) shutil.copy(python_exe, "dist/vuegen_gui/_internal/python") print("Done.") From 068239ad9fdcde7c5915eec4f925b2832780e578 Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Thu, 6 Mar 2025 07:47:47 +0100 Subject: [PATCH 39/91] :construction: move python file to main folder --- gui/copy_python_executable.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gui/copy_python_executable.py b/gui/copy_python_executable.py index 740bf6e..6bee635 100644 --- a/gui/copy_python_executable.py +++ b/gui/copy_python_executable.py @@ -4,5 +4,5 @@ python_exe = Path(sys.executable) # .with_stem("python") print("Copying python executable:", python_exe) -shutil.copy(python_exe, "dist/vuegen_gui/_internal/python") +shutil.copy(python_exe, "dist/vuegen_gui/python") print("Done.") From 4e93792aa486c9e012c70b432f57f96dad347510 Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Thu, 6 Mar 2025 09:27:57 +0100 Subject: [PATCH 40/91] =?UTF-8?q?=F0=9F=9A=A7=20try=20to=20use=20miniconda?= =?UTF-8?q?=20action=20for=20python=20exe?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/cdci.yml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/cdci.yml b/.github/workflows/cdci.yml index f860d9c..3ceb979 100644 --- a/.github/workflows/cdci.yml +++ b/.github/workflows/cdci.yml @@ -126,9 +126,9 @@ jobs: build-executable: name: Build-exe-${{ matrix.os.label }} runs-on: ${{ matrix.os.runner }} - needs: - - test - - other-reports + # needs: + # - test + # - other-reports strategy: matrix: python-version: ["3.11"] @@ -144,8 +144,12 @@ jobs: label: "windows-x64" steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + # - uses: actions/setup-python@v5 + # with: + # python-version: ${{ matrix.python-version }} + - uses: conda-incubator/setup-miniconda@v3 with: + auto-update-conda: true python-version: ${{ matrix.python-version }} - name: Install VueGen GUI and pyinstaller run: python -m pip install ".[gui]" pyinstaller From 9f908148a23a0cc79c6e1ed0d401f609e4d54fc5 Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Thu, 6 Mar 2025 10:46:34 +0100 Subject: [PATCH 41/91] =?UTF-8?q?=F0=9F=9A=A7=20inspect=20conda=20on=20Git?= =?UTF-8?q?Hub=20Actions=20runners?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/cdci.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cdci.yml b/.github/workflows/cdci.yml index 3ceb979..2fb8b20 100644 --- a/.github/workflows/cdci.yml +++ b/.github/workflows/cdci.yml @@ -152,11 +152,16 @@ jobs: auto-update-conda: true python-version: ${{ matrix.python-version }} - name: Install VueGen GUI and pyinstaller - run: python -m pip install ".[gui]" pyinstaller + run: | + conda info + conda list + python -m pip install ".[gui]" pyinstaller + conda list - name: Build executable run: | cd gui pyinstaller -n vuegen_gui -D --collect-all pyvis --collect-all streamlit --collect-all st_aggrid --collect-all customtkinter --collect-all quarto_cli --collect-all jupyter_core --collect-all yaml --collect-all ipykernel --collect-all nbconvert --collect-all notebook --collect-all ipywidgets --collect-all jupyter_console --collect-all jupyter_client --add-data ../docs/example_data/Basic_example_vuegen_demo_notebook:example_data/Basic_example_vuegen_demo_notebook app.py + conda info python copy_python_executable.py - name: Upload executable uses: actions/upload-artifact@v4 From 40244eacb251f912c5f2b445ebe1639280cd8bca Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Thu, 6 Mar 2025 10:53:54 +0100 Subject: [PATCH 42/91] =?UTF-8?q?=F0=9F=9A=A7=20find=20the=20documented=20?= =?UTF-8?q?test=20environment?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/cdci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/cdci.yml b/.github/workflows/cdci.yml index 2fb8b20..5d1ae2d 100644 --- a/.github/workflows/cdci.yml +++ b/.github/workflows/cdci.yml @@ -151,9 +151,11 @@ jobs: with: auto-update-conda: true python-version: ${{ matrix.python-version }} + auto-activate-base: true - name: Install VueGen GUI and pyinstaller run: | conda info + conda info -e conda list python -m pip install ".[gui]" pyinstaller conda list From d89adf2ae349cf9481a76ebb84d5e032ff90a1bb Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Thu, 6 Mar 2025 11:03:09 +0100 Subject: [PATCH 43/91] :bug: test environment not auto-activated --- .github/workflows/cdci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/cdci.yml b/.github/workflows/cdci.yml index 5d1ae2d..5792c5a 100644 --- a/.github/workflows/cdci.yml +++ b/.github/workflows/cdci.yml @@ -152,6 +152,7 @@ jobs: auto-update-conda: true python-version: ${{ matrix.python-version }} auto-activate-base: true + activate-environment: "test" - name: Install VueGen GUI and pyinstaller run: | conda info From 6736e37ede08eeaab7678711b28a06b042f62f02 Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Thu, 6 Mar 2025 11:08:10 +0100 Subject: [PATCH 44/91] =?UTF-8?q?=F0=9F=9A=A7=20try=20to=20activate=20test?= =?UTF-8?q?=20env...?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/cdci.yml | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/.github/workflows/cdci.yml b/.github/workflows/cdci.yml index 5792c5a..ab8f3f4 100644 --- a/.github/workflows/cdci.yml +++ b/.github/workflows/cdci.yml @@ -134,14 +134,14 @@ jobs: python-version: ["3.11"] os: # https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/running-variations-of-jobs-in-a-workflow#example-using-a-multi-dimension-matrix - - runner: "ubuntu-latest" - label: "ubuntu-latest_LTS-x64" - - runner: "macos-13" - label: "macos-13-x64" + # - runner: "ubuntu-latest" + # label: "ubuntu-latest_LTS-x64" + # - runner: "macos-13" + # label: "macos-13-x64" - runner: "macos-15" label: "macos-15-arm64" - - runner: "windows-latest" - label: "windows-x64" + # - runner: "windows-latest" + # label: "windows-x64" steps: - uses: actions/checkout@v4 # - uses: actions/setup-python@v5 @@ -151,17 +151,18 @@ jobs: with: auto-update-conda: true python-version: ${{ matrix.python-version }} - auto-activate-base: true activate-environment: "test" - name: Install VueGen GUI and pyinstaller run: | conda info conda info -e + conda activate test conda list python -m pip install ".[gui]" pyinstaller conda list - name: Build executable run: | + conda activate test cd gui pyinstaller -n vuegen_gui -D --collect-all pyvis --collect-all streamlit --collect-all st_aggrid --collect-all customtkinter --collect-all quarto_cli --collect-all jupyter_core --collect-all yaml --collect-all ipykernel --collect-all nbconvert --collect-all notebook --collect-all ipywidgets --collect-all jupyter_console --collect-all jupyter_client --add-data ../docs/example_data/Basic_example_vuegen_demo_notebook:example_data/Basic_example_vuegen_demo_notebook app.py conda info From 866616ceffa0dc93be227ef03dac312e2dbb5579 Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Thu, 6 Mar 2025 11:20:59 +0100 Subject: [PATCH 45/91] =?UTF-8?q?=F0=9F=9A=A7=20get=20environment=20to=20b?= =?UTF-8?q?e=20activated?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/cdci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cdci.yml b/.github/workflows/cdci.yml index ab8f3f4..6ac6502 100644 --- a/.github/workflows/cdci.yml +++ b/.github/workflows/cdci.yml @@ -149,13 +149,15 @@ jobs: # python-version: ${{ matrix.python-version }} - uses: conda-incubator/setup-miniconda@v3 with: - auto-update-conda: true python-version: ${{ matrix.python-version }} activate-environment: "test" + auto-activate-base: true + auto-update-conda: true - name: Install VueGen GUI and pyinstaller run: | conda info conda info -e + conda init conda activate test conda list python -m pip install ".[gui]" pyinstaller From 03af4798668d7f3ab2e88e7e0ebf5feac52fdf08 Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Thu, 6 Mar 2025 11:23:23 +0100 Subject: [PATCH 46/91] =?UTF-8?q?=F0=9F=9A=A7=20modify=20default=20shell?= =?UTF-8?q?=20params?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/cdci.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cdci.yml b/.github/workflows/cdci.yml index 6ac6502..3a42205 100644 --- a/.github/workflows/cdci.yml +++ b/.github/workflows/cdci.yml @@ -126,6 +126,9 @@ jobs: build-executable: name: Build-exe-${{ matrix.os.label }} runs-on: ${{ matrix.os.runner }} + defaults: + run: + shell: bash -el {0} # needs: # - test # - other-reports @@ -157,8 +160,6 @@ jobs: run: | conda info conda info -e - conda init - conda activate test conda list python -m pip install ".[gui]" pyinstaller conda list From 78749923858ba1fba3cdc2ec2797e480c79f9d4c Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Thu, 6 Mar 2025 13:58:36 +0100 Subject: [PATCH 47/91] =?UTF-8?q?=F0=9F=9A=A7=20use=20quarto=20native=20fu?= =?UTF-8?q?nctionality?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - require users to have a Python base installation with jupyter installed - pip install jupyter Then quarto manages to pick up local dependencies and run the notebook (to test) --- .github/workflows/cdci.yml | 20 +---- src/vuegen/quarto_reportview.py | 133 ++++++++++++++++++-------------- 2 files changed, 75 insertions(+), 78 deletions(-) diff --git a/.github/workflows/cdci.yml b/.github/workflows/cdci.yml index 3a42205..a938dae 100644 --- a/.github/workflows/cdci.yml +++ b/.github/workflows/cdci.yml @@ -126,9 +126,6 @@ jobs: build-executable: name: Build-exe-${{ matrix.os.label }} runs-on: ${{ matrix.os.runner }} - defaults: - run: - shell: bash -el {0} # needs: # - test # - other-reports @@ -147,29 +144,16 @@ jobs: # label: "windows-x64" steps: - uses: actions/checkout@v4 - # - uses: actions/setup-python@v5 - # with: - # python-version: ${{ matrix.python-version }} - - uses: conda-incubator/setup-miniconda@v3 + - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - activate-environment: "test" - auto-activate-base: true - auto-update-conda: true - name: Install VueGen GUI and pyinstaller run: | - conda info - conda info -e - conda list python -m pip install ".[gui]" pyinstaller - conda list - name: Build executable run: | - conda activate test cd gui - pyinstaller -n vuegen_gui -D --collect-all pyvis --collect-all streamlit --collect-all st_aggrid --collect-all customtkinter --collect-all quarto_cli --collect-all jupyter_core --collect-all yaml --collect-all ipykernel --collect-all nbconvert --collect-all notebook --collect-all ipywidgets --collect-all jupyter_console --collect-all jupyter_client --add-data ../docs/example_data/Basic_example_vuegen_demo_notebook:example_data/Basic_example_vuegen_demo_notebook app.py - conda info - python copy_python_executable.py + pyinstaller -n vuegen_gui -D --collect-all pyvis --collect-all streamlit --collect-all st_aggrid --collect-all customtkinter --collect-all quarto_cli --add-data ../docs/example_data/Basic_example_vuegen_demo_notebook:example_data/Basic_example_vuegen_demo_notebook app.py - name: Upload executable uses: actions/upload-artifact@v4 with: diff --git a/src/vuegen/quarto_reportview.py b/src/vuegen/quarto_reportview.py index 1286a0b..69e57ca 100644 --- a/src/vuegen/quarto_reportview.py +++ b/src/vuegen/quarto_reportview.py @@ -23,10 +23,12 @@ class QuartoReportView(r.ReportView): def __init__(self, report: r.Report, report_type: r.ReportType): super().__init__(report=report, report_type=report_type) self.BUNDLED_EXECUTION = False + self.quarto_path = "quarto" if getattr(sys, "frozen", False) and hasattr(sys, "_MEIPASS"): self.report.logger.info("running in a PyInstaller bundle") self.BUNDLED_EXECUTION = True self.report.logger.debug(f"sys._MEIPASS: {sys._MEIPASS}") + self.quarto_path = Path(sys._MEIPASS) / "quarto_cli" / "bin" / "quarto" else: self.report.logger.info("running in a normal Python process") @@ -164,74 +166,85 @@ def run_report(self, output_dir: str = BASE_DIR) -> None: # from quarto_cli import run_quarto # entrypoint of quarto-cli not in module? file_path_to_qmd = os.path.join(output_dir, f"{self.BASE_DIR}.qmd") - args = ["quarto", "render", file_path_to_qmd] + args = [self.quarto_path, "render", file_path_to_qmd] self.report.logger.info( f"Running '{self.report.title}' '{self.report_type}' report with {args!r}" ) - if not self.BUNDLED_EXECUTION: - subprocess.run( - ["quarto", "render", os.path.join(output_dir, f"{self.BASE_DIR}.qmd")], - check=True, - ) - try: - if self.report_type == r.ReportType.JUPYTER: - args = ["quarto", "convert", file_path_to_qmd] - subprocess.run( - args, - check=True, - ) - self.report.logger.info( - f"Converted '{self.report.title}' '{self.report_type}' report to Jupyter Notebook after execution" - ) - self.report.logger.info( - f"'{self.report.title}' '{self.report_type}' report rendered" - ) - except subprocess.CalledProcessError as e: - self.report.logger.error( - f"Error running '{self.report.title}' {self.report_type} report: {str(e)}" + subprocess.run( + [ + self.quarto_path, + "render", + os.path.join(output_dir, f"{self.BASE_DIR}.qmd"), + ], + check=True, + ) + try: + if self.report_type == r.ReportType.JUPYTER: + args = [self.quarto_path, "convert", file_path_to_qmd] + subprocess.run( + args, + check=True, ) - raise - except FileNotFoundError as e: - self.report.logger.error( - f"Quarto is not installed. Please install Quarto to run the report: {str(e)}" + self.report.logger.info( + f"Converted '{self.report.title}' '{self.report_type}' report to Jupyter Notebook after execution" ) - raise - else: - quarto_path = Path(sys._MEIPASS) / "quarto_cli" / "bin" / "quarto" - _sys_exe = sys.executable - # set executable to the bundled python (manually added to bundle) - sys.executable = str(Path(sys._MEIPASS).parent / "python") - self.report.logger.info(f"quarto_path: {quarto_path}") - - args = [f"{quarto_path}", "convert", file_path_to_qmd] - subprocess.run( - args, - check=True, - ) self.report.logger.info( - f"Converted '{self.report.title}' '{self.report_type}' report to Jupyter Notebook after execution" + f"'{self.report.title}' '{self.report_type}' report rendered" ) - notebook_filename = Path(file_path_to_qmd).with_suffix(".ipynb") - # quarto does not try to execute ipynb files, just render them - # execute manually using bundled python binary - # https://nbconvert.readthedocs.io/en/latest/execute_api.html - import nbformat - from nbconvert.preprocessors import ExecutePreprocessor - - with open(notebook_filename, encoding="utf-8") as f: - nb = nbformat.read(f, as_version=4) - logging.getLogger("traitlets").setLevel(logging.INFO) - ep = ExecutePreprocessor(timeout=600, kernel_name="python3") - nb, _ = ep.preprocess(nb, {"metadata": {"path": "./"}}) - with open(notebook_filename, "w", encoding="utf-8") as f: - nbformat.write(nb, f) - # quarto does not try execute ipynb files per default, just render these - args = [f"{quarto_path}", "render", notebook_filename] - subprocess.run( - args, - check=True, + except subprocess.CalledProcessError as e: + self.report.logger.error( + f"Error running '{self.report.title}' {self.report_type} report: {str(e)}" ) - sys.executable = _sys_exe + raise + except FileNotFoundError as e: + self.report.logger.error( + f"Quarto is not installed. Please install Quarto to run the report: {str(e)}" + ) + raise + # else: + # quarto_path = Path(sys._MEIPASS) / "quarto_cli" / "bin" / "quarto" + # _sys_exe = sys.executable + # set executable to the bundled python (manually added to bundle) + # sys.executable = str(Path(sys._MEIPASS).parent / "python") + # self.report.logger.info(f"quarto_path: {self.quarto_path}") + + # args = [f"{quarto_path}", "render", file_path_to_qmd] + # subprocess.run( + # args, + # check=True, + # ) + # self.report.logger.info( + # f"Converted '{self.report.title}' '{self.report_type}' report to Jupyter Notebook after execution" + # ) + # notebook_filename = Path(file_path_to_qmd).with_suffix(".ipynb") + # try papermill + # import papermill as pm + + # pm.execute_notebook( + # "path/to/input.ipynb", + # "path/to/output.ipynb", + # parameters=dict(alpha=0.6, ratio=0.1), + # ) + # quarto does not try to execute ipynb files, just render them + # execute manually using bundled python binary + # https://nbconvert.readthedocs.io/en/latest/execute_api.html + # import nbformat + # from nbconvert.preprocessors import ExecutePreprocessor + + # with open(notebook_filename, encoding="utf-8") as f: + # nb = nbformat.read(f, as_version=4) + # logging.getLogger("traitlets").setLevel(logging.INFO) + # ep = ExecutePreprocessor(timeout=600, kernel_name="python3") + # nb, _ = ep.preprocess(nb, {"metadata": {"path": "./"}}) + # with open(notebook_filename, "w", encoding="utf-8") as f: + # nbformat.write(nb, f) + # quarto does not try execute ipynb files per default, just render these + # args = [f"{quarto_path}", "render", notebook_filename] + # subprocess.run( + # args, + # check=True, + # ) + # sys.executable = _sys_exe def _create_yaml_header(self) -> str: """ From 5a36a0e13eeb3e3db07dfdf9c0e2f7738f6f5f92 Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Thu, 6 Mar 2025 15:40:00 +0100 Subject: [PATCH 48/91] =?UTF-8?q?=F0=9F=9A=A7=20try=20to=20add=20tinytex?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/cdci.yml | 3 +++ README.md | 19 +++++++++++++- gui/copy_python_executable.py | 8 ------ src/vuegen/quarto_reportview.py | 44 --------------------------------- 4 files changed, 21 insertions(+), 53 deletions(-) delete mode 100644 gui/copy_python_executable.py diff --git a/.github/workflows/cdci.yml b/.github/workflows/cdci.yml index a938dae..cf38124 100644 --- a/.github/workflows/cdci.yml +++ b/.github/workflows/cdci.yml @@ -150,6 +150,9 @@ jobs: - name: Install VueGen GUI and pyinstaller run: | python -m pip install ".[gui]" pyinstaller + - name: Install tinytex into local quarto installation + run: | + quarto install tinytex - name: Build executable run: | cd gui diff --git a/README.md b/README.md index 816c1f3..5d886b6 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ Also, the class diagram for the project is presented below to illustrate the arc An extended version of the class diagram with attributes and methods is available [here][vuegen-class-diag-att]. -The VueGen documentation is available at [vuegen.readthedocs.io][vuegen-docs], where you can find detailed information of the package’s classes and functions, installation and execution instructions, and case studies to demonstrate its functionality. +The VueGen documentation is available at [vuegen.readthedocs.io][vuegen-docs], where you can find detailed information of the package’s classes and functions, installation and execution instructions, and case studies to demonstrate its functionality. ## Installation Vuegen is available on [PyPI][vuegen-pypi] and can be installed using pip: @@ -102,6 +102,23 @@ docker run --rm \ quay.io/dtu_biosustain_dsp/vuegen:docker --directory /home/appuser/Earth_microbiome_vuegen_demo_notebook --report_type streamlit ``` +## GUI + +We have a simple GUI for VueGen that can be run locally or through a standalone executable. + +```bash +cd gui +python app.py +``` + +The bundle GUI with the VueGen package is available under the releases. You will need to +unzip the file and run `vuegen_gui` in the unpacked main folder. Most dependencies are included into +the bundle under `_internals` using PyInstaller. + +Streamlit works out of the box as a purely Python based package. For `html` creation you will have to +have a global Python installation with the `jupyter` package installed. `quarto` needs to start +a kernel for execution. This is also true if you install `quarto` globally on your machine. + ## Case studies VueGen’s functionality is demonstrated through two case studies: diff --git a/gui/copy_python_executable.py b/gui/copy_python_executable.py deleted file mode 100644 index 6bee635..0000000 --- a/gui/copy_python_executable.py +++ /dev/null @@ -1,8 +0,0 @@ -import shutil -import sys -from pathlib import Path - -python_exe = Path(sys.executable) # .with_stem("python") -print("Copying python executable:", python_exe) -shutil.copy(python_exe, "dist/vuegen_gui/python") -print("Done.") diff --git a/src/vuegen/quarto_reportview.py b/src/vuegen/quarto_reportview.py index 69e57ca..152a4a6 100644 --- a/src/vuegen/quarto_reportview.py +++ b/src/vuegen/quarto_reportview.py @@ -201,50 +201,6 @@ def run_report(self, output_dir: str = BASE_DIR) -> None: f"Quarto is not installed. Please install Quarto to run the report: {str(e)}" ) raise - # else: - # quarto_path = Path(sys._MEIPASS) / "quarto_cli" / "bin" / "quarto" - # _sys_exe = sys.executable - # set executable to the bundled python (manually added to bundle) - # sys.executable = str(Path(sys._MEIPASS).parent / "python") - # self.report.logger.info(f"quarto_path: {self.quarto_path}") - - # args = [f"{quarto_path}", "render", file_path_to_qmd] - # subprocess.run( - # args, - # check=True, - # ) - # self.report.logger.info( - # f"Converted '{self.report.title}' '{self.report_type}' report to Jupyter Notebook after execution" - # ) - # notebook_filename = Path(file_path_to_qmd).with_suffix(".ipynb") - # try papermill - # import papermill as pm - - # pm.execute_notebook( - # "path/to/input.ipynb", - # "path/to/output.ipynb", - # parameters=dict(alpha=0.6, ratio=0.1), - # ) - # quarto does not try to execute ipynb files, just render them - # execute manually using bundled python binary - # https://nbconvert.readthedocs.io/en/latest/execute_api.html - # import nbformat - # from nbconvert.preprocessors import ExecutePreprocessor - - # with open(notebook_filename, encoding="utf-8") as f: - # nb = nbformat.read(f, as_version=4) - # logging.getLogger("traitlets").setLevel(logging.INFO) - # ep = ExecutePreprocessor(timeout=600, kernel_name="python3") - # nb, _ = ep.preprocess(nb, {"metadata": {"path": "./"}}) - # with open(notebook_filename, "w", encoding="utf-8") as f: - # nbformat.write(nb, f) - # quarto does not try execute ipynb files per default, just render these - # args = [f"{quarto_path}", "render", notebook_filename] - # subprocess.run( - # args, - # check=True, - # ) - # sys.executable = _sys_exe def _create_yaml_header(self) -> str: """ From 743221cdc5a1b6a7e2d2c683f42b86a196cc10b0 Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Thu, 6 Mar 2025 16:31:14 +0100 Subject: [PATCH 49/91] :bug: check for dependencies --- src/vuegen/quarto_reportview.py | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/src/vuegen/quarto_reportview.py b/src/vuegen/quarto_reportview.py index 152a4a6..aa8b764 100644 --- a/src/vuegen/quarto_reportview.py +++ b/src/vuegen/quarto_reportview.py @@ -170,15 +170,28 @@ def run_report(self, output_dir: str = BASE_DIR) -> None: self.report.logger.info( f"Running '{self.report.title}' '{self.report_type}' report with {args!r}" ) - subprocess.run( - [ - self.quarto_path, - "render", - os.path.join(output_dir, f"{self.BASE_DIR}.qmd"), - ], - check=True, - ) + if self.report_type in [ + r.ReportType.PDF, + r.ReportType.DOCX, + r.ReportType.ODT, + ]: + subprocess.run( + [self.quarto_path, "install", "tinytex"], + check=True, + ) + subprocess.run( + [self.quarto_path, "install", "chromium"], + check=True, + ) try: + subprocess.run( + [ + self.quarto_path, + "render", + os.path.join(output_dir, f"{self.BASE_DIR}.qmd"), + ], + check=True, + ) if self.report_type == r.ReportType.JUPYTER: args = [self.quarto_path, "convert", file_path_to_qmd] subprocess.run( From 668c2450a4682c3a2e5fc2b008137307d4b04250 Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Fri, 7 Mar 2025 08:39:12 +0100 Subject: [PATCH 50/91] :bug: try to add both tools on the fly (chromnium and tinyte) needed for pdfs, docx, pptx, etc --- .github/workflows/cdci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cdci.yml b/.github/workflows/cdci.yml index cf38124..b63dc7e 100644 --- a/.github/workflows/cdci.yml +++ b/.github/workflows/cdci.yml @@ -150,8 +150,9 @@ jobs: - name: Install VueGen GUI and pyinstaller run: | python -m pip install ".[gui]" pyinstaller - - name: Install tinytex into local quarto installation + - name: Install quarto tools run: | + quarto install chromium quarto install tinytex - name: Build executable run: | From ac13935e5083f2bde3dc3a5fb8f8ce0dcc3afd21 Mon Sep 17 00:00:00 2001 From: enryh Date: Fri, 7 Mar 2025 13:24:35 +0100 Subject: [PATCH 51/91] :bug: close parantheses (merge error) --- src/vuegen/quarto_reportview.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/vuegen/quarto_reportview.py b/src/vuegen/quarto_reportview.py index d383979..4321ed4 100644 --- a/src/vuegen/quarto_reportview.py +++ b/src/vuegen/quarto_reportview.py @@ -165,7 +165,7 @@ def run_report(self, output_dir: str = BASE_DIR) -> None: """ # from quarto_cli import run_quarto # entrypoint of quarto-cli not in module? - file_path_to_qmd = str(Path(output_dir) / f"{self.BASE_DIR}.qmd" + file_path_to_qmd = str(Path(output_dir) / f"{self.BASE_DIR}.qmd") args = [self.quarto_path, "render", file_path_to_qmd] self.report.logger.info( f"Running '{self.report.title}' '{self.report_type}' report with {args!r}" From 012526f215e8e24fa5b8b53d2fc454288bab0ff5 Mon Sep 17 00:00:00 2001 From: enryh Date: Fri, 7 Mar 2025 13:55:03 +0100 Subject: [PATCH 52/91] :bug: make Windows Path for bundled quarto executable - install quarto on server (as it seems to fail in conda envs which I use locally) --- .github/workflows/cdci.yml | 4 ++-- src/vuegen/quarto_reportview.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/cdci.yml b/.github/workflows/cdci.yml index b63dc7e..095f71f 100644 --- a/.github/workflows/cdci.yml +++ b/.github/workflows/cdci.yml @@ -140,8 +140,8 @@ jobs: # label: "macos-13-x64" - runner: "macos-15" label: "macos-15-arm64" - # - runner: "windows-latest" - # label: "windows-x64" + - runner: "windows-latest" + label: "windows-x64" steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 diff --git a/src/vuegen/quarto_reportview.py b/src/vuegen/quarto_reportview.py index 4321ed4..17f11ce 100644 --- a/src/vuegen/quarto_reportview.py +++ b/src/vuegen/quarto_reportview.py @@ -28,7 +28,7 @@ def __init__(self, report: r.Report, report_type: r.ReportType): self.report.logger.info("running in a PyInstaller bundle") self.BUNDLED_EXECUTION = True self.report.logger.debug(f"sys._MEIPASS: {sys._MEIPASS}") - self.quarto_path = Path(sys._MEIPASS) / "quarto_cli" / "bin" / "quarto" + self.quarto_path = str(Path(sys._MEIPASS) / "quarto_cli" / "bin" / "quarto") else: self.report.logger.info("running in a normal Python process") From f52e8c6613752c091f323597a3a7e291a7c7242a Mon Sep 17 00:00:00 2001 From: enryh Date: Fri, 7 Mar 2025 14:51:12 +0100 Subject: [PATCH 53/91] =?UTF-8?q?=F0=9F=9A=A7=20test=20further=20options?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/cdci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cdci.yml b/.github/workflows/cdci.yml index 095f71f..47bfa95 100644 --- a/.github/workflows/cdci.yml +++ b/.github/workflows/cdci.yml @@ -157,7 +157,7 @@ jobs: - name: Build executable run: | cd gui - pyinstaller -n vuegen_gui -D --collect-all pyvis --collect-all streamlit --collect-all st_aggrid --collect-all customtkinter --collect-all quarto_cli --add-data ../docs/example_data/Basic_example_vuegen_demo_notebook:example_data/Basic_example_vuegen_demo_notebook app.py + pyinstaller -n vuegen_gui -D -w --collect-all pyvis --collect-all streamlit --collect-all st_aggrid --collect-all customtkinter --collect-all quarto_cli --add-data ../docs/example_data/Basic_example_vuegen_demo_notebook:example_data/Basic_example_vuegen_demo_notebook app.py - name: Upload executable uses: actions/upload-artifact@v4 with: From 0196926c9df4d513e2fe0dbe366639ee9ed6cdc0 Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Sat, 8 Mar 2025 10:17:26 +0100 Subject: [PATCH 54/91] :art: add file directory selection - allow to specify a directory - added Alberto's directory selection dialog - added the report generator functionality as suggested by Alberto - success needs to be printed manually. quarto errors are not propagated to our app... (exist code of subprocess run) --- gui/app.py | 65 ++++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 51 insertions(+), 14 deletions(-) diff --git a/gui/app.py b/gui/app.py index 3f5d8a4..fed9bb9 100644 --- a/gui/app.py +++ b/gui/app.py @@ -22,14 +22,19 @@ import sys import tkinter as tk from pathlib import Path +from pprint import pprint import customtkinter -from vuegen.__main__ import main +# from vuegen.__main__ import main from vuegen.report import ReportType customtkinter.set_appearance_mode("system") customtkinter.set_default_color_theme("dark-blue") +from tkinter import filedialog + +from vuegen import report_generator +from vuegen.utils import print_completion_message app_path = Path(__file__).absolute() print("app_path:", app_path) @@ -56,23 +61,42 @@ ########################################################################################## # callbacks +# def create_run_vuegen(is_dir, config_path, report_type, run_streamlit): +# def inner(): +# args = ["vuegen"] +# print(f"{is_dir.get() = }") +# if is_dir.get(): +# args.append("--directory") +# else: +# args.append("--config") +# args.append(config_path.get()) +# args.append("--report_type") +# args.append(report_type.get()) +# print(f"{run_streamlit.get() = }") +# if run_streamlit.get(): +# args.append("--streamlit_autorun") +# print("args:", args) +# sys.argv = args +# main() # Call the main function from vuegen + +# return inner + + def create_run_vuegen(is_dir, config_path, report_type, run_streamlit): def inner(): - args = ["vuegen"] + kwargs = {} print(f"{is_dir.get() = }") if is_dir.get(): - args.append("--directory") + kwargs["dir_path"] = config_path.get() else: - args.append("--config") - args.append(config_path.get()) - args.append("--report_type") - args.append(report_type.get()) + kwargs["config_path"] = config_path.get() + kwargs["report_type"] = report_type.get() print(f"{run_streamlit.get() = }") - if run_streamlit.get(): - args.append("--streamlit_autorun") - print("args:", args) - sys.argv = args - main() # Call the main function from vuegen + kwargs["streamlit_autorun"] = run_streamlit.get() + print("kwargs:") + pprint(kwargs) + report_generator.get_report(**kwargs) + print_completion_message(report_type.get()) return inner @@ -89,10 +113,18 @@ def radio_button_callback(): return radio_button_callback +def create_select_directory(string_var): + def select_directory(): + directory = filedialog.askdirectory() + string_var.set(directory) + + return select_directory + + ########################################################################################## # APP app = customtkinter.CTk() -app.geometry("460x400") +app.geometry("600x400") app.title("VueGen GUI") ########################################################################################## @@ -127,7 +159,12 @@ def radio_button_callback(): width=400, textvariable=config_path, ) -config_path_entry.grid(row=2, column=0, columnspan=2, padx=20, pady=10) +config_path_entry.grid(row=2, column=0, columnspan=2, padx=5, pady=10) +select_directory = create_select_directory(config_path) +select_button = customtkinter.CTkButton( + app, text="Select Directory", command=select_directory +) +select_button.grid(row=2, column=2, columnspan=2, padx=5, pady=10) ########################################################################################## # Report type dropdown From 62a6c69975573d38acd98bafb6461fddba8ff923 Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Sat, 8 Mar 2025 11:24:05 +0100 Subject: [PATCH 55/91] =?UTF-8?q?=F0=9F=9A=A7=20test=20now=20with=20onefil?= =?UTF-8?q?e?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - importing vuegen.generate_report increased the MAC OS file size - or it was the windowed option? --- .github/workflows/cdci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cdci.yml b/.github/workflows/cdci.yml index 47bfa95..2b1e8db 100644 --- a/.github/workflows/cdci.yml +++ b/.github/workflows/cdci.yml @@ -157,7 +157,7 @@ jobs: - name: Build executable run: | cd gui - pyinstaller -n vuegen_gui -D -w --collect-all pyvis --collect-all streamlit --collect-all st_aggrid --collect-all customtkinter --collect-all quarto_cli --add-data ../docs/example_data/Basic_example_vuegen_demo_notebook:example_data/Basic_example_vuegen_demo_notebook app.py + pyinstaller -n vuegen_gui -D -w -F --collect-all pyvis --collect-all streamlit --collect-all st_aggrid --collect-all customtkinter --collect-all quarto_cli --add-data ../docs/example_data/Basic_example_vuegen_demo_notebook:example_data/Basic_example_vuegen_demo_notebook app.py - name: Upload executable uses: actions/upload-artifact@v4 with: From 830b5574aafc80422c83b5544b265c1791d570ae Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Sat, 8 Mar 2025 11:36:49 +0100 Subject: [PATCH 56/91] =?UTF-8?q?=F0=9F=9A=A7=20test=20also=20non-windowed?= =?UTF-8?q?=20app?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/cdci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cdci.yml b/.github/workflows/cdci.yml index 2b1e8db..728e223 100644 --- a/.github/workflows/cdci.yml +++ b/.github/workflows/cdci.yml @@ -157,7 +157,8 @@ jobs: - name: Build executable run: | cd gui - pyinstaller -n vuegen_gui -D -w -F --collect-all pyvis --collect-all streamlit --collect-all st_aggrid --collect-all customtkinter --collect-all quarto_cli --add-data ../docs/example_data/Basic_example_vuegen_demo_notebook:example_data/Basic_example_vuegen_demo_notebook app.py + pyinstaller -n vuegen_gui -D --collect-all pyvis --collect-all streamlit --collect-all st_aggrid --collect-all customtkinter --collect-all quarto_cli --add-data ../docs/example_data/Basic_example_vuegen_demo_notebook:example_data/Basic_example_vuegen_demo_notebook app.py + # -w -F - name: Upload executable uses: actions/upload-artifact@v4 with: From 9ae107462856595ba575560d5ed07a5bfb45c85a Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Sat, 8 Mar 2025 12:37:30 +0100 Subject: [PATCH 57/91] :bug: set default static dir only on main methods, pass parameter on other hidden methods - will allow to set static dir on app or cli --- src/vuegen/quarto_reportview.py | 43 +++++++++++++++++++++--------- src/vuegen/streamlit_reportview.py | 28 ++++++++++++------- 2 files changed, 49 insertions(+), 22 deletions(-) diff --git a/src/vuegen/quarto_reportview.py b/src/vuegen/quarto_reportview.py index 17f11ce..bb17606 100644 --- a/src/vuegen/quarto_reportview.py +++ b/src/vuegen/quarto_reportview.py @@ -113,7 +113,10 @@ def generate_report( # Generate content for the subsection subsection_content, subsection_imports = ( self._generate_subsection( - subsection, is_report_static, is_report_revealjs + subsection, + is_report_static, + is_report_revealjs, + static_dir=static_dir, ) ) qmd_content.extend(subsection_content) @@ -352,7 +355,11 @@ def _create_yaml_header(self) -> str: return yaml_header def _generate_subsection( - self, subsection, is_report_static, is_report_revealjs + self, + subsection, + is_report_static, + is_report_revealjs, + static_dir: str, ) -> tuple[List[str], List[str]]: """ Generate code to render components (plots, dataframes, markdown) in the given subsection, @@ -366,6 +373,8 @@ def _generate_subsection( A boolean indicating whether the report is static or interactive. is_report_revealjs : bool A boolean indicating whether the report is in revealjs format. + static_dir : str + The folder where the static files will be saved. Returns ------- tuple : (List[str], List[str]) @@ -389,11 +398,15 @@ def _generate_subsection( if component.component_type == r.ComponentType.PLOT: subsection_content.extend( - self._generate_plot_content(component, is_report_static) + self._generate_plot_content( + component, is_report_static, static_dir=static_dir + ) ) elif component.component_type == r.ComponentType.DATAFRAME: subsection_content.extend( - self._generate_dataframe_content(component, is_report_static) + self._generate_dataframe_content( + component, is_report_static, static_dir=static_dir + ) ) elif ( component.component_type == r.ComponentType.MARKDOWN @@ -419,7 +432,7 @@ def _generate_subsection( return subsection_content, subsection_imports def _generate_plot_content( - self, plot, is_report_static, static_dir: str = STATIC_FILES_DIR + self, plot, is_report_static, static_dir: str ) -> List[str]: """ Generate content for a plot component based on the report type. @@ -428,8 +441,8 @@ def _generate_plot_content( ---------- plot : Plot The plot component to generate content for. - static_dir : str, optional - The folder where the static files will be saved (default is STATIC_FILES_DIR). + static_dir : str + The folder where the static files will be saved. Returns ------- @@ -561,7 +574,9 @@ def _generate_plot_code(self, plot, output_file="") -> str: \n""" return plot_code - def _generate_dataframe_content(self, dataframe, is_report_static) -> List[str]: + def _generate_dataframe_content( + self, dataframe, is_report_static, static_dir: str + ) -> List[str]: """ Generate content for a DataFrame component based on the report type. @@ -571,6 +586,8 @@ def _generate_dataframe_content(self, dataframe, is_report_static) -> List[str]: The dataframe component to add to content. is_report_static : bool A boolean indicating whether the report is static or interactive. + static_dir : str + The folder where the static files will be saved. Returns ------- @@ -620,7 +637,9 @@ def _generate_dataframe_content(self, dataframe, is_report_static) -> List[str]: ) # Display the dataframe - dataframe_content.extend(self._show_dataframe(dataframe, is_report_static)) + dataframe_content.extend( + self._show_dataframe(dataframe, is_report_static, static_dir=static_dir) + ) except Exception as e: self.report.logger.error( @@ -772,7 +791,7 @@ def _generate_image_content( ) def _show_dataframe( - self, dataframe, is_report_static, static_dir: str = STATIC_FILES_DIR + self, dataframe, is_report_static, static_dir: str ) -> List[str]: """ Appends either a static image or an interactive representation of a DataFrame to the content list. @@ -783,8 +802,8 @@ def _show_dataframe( The DataFrame object containing the data to display. is_report_static : bool Determines if the report is in a static format (e.g., PDF) or interactive (e.g., HTML). - static_dir : str, optional - The folder where the static files will be saved (default is STATIC_FILES_DIR). + static_dir : str + The folder where the static files will be saved. Returns ------- diff --git a/src/vuegen/streamlit_reportview.py b/src/vuegen/streamlit_reportview.py index de4738f..63f7553 100644 --- a/src/vuegen/streamlit_reportview.py +++ b/src/vuegen/streamlit_reportview.py @@ -144,7 +144,7 @@ def generate_report( ) # Create Python files for each section and its subsections and plots - self._generate_sections(output_dir=output_dir) + self._generate_sections(output_dir=output_dir, static_dir=static_dir) except Exception as e: self.report.logger.error( f"An error occurred while generating the report: {str(e)}" @@ -308,7 +308,7 @@ def _generate_home_section( self.report.logger.error(f"Error generating the home section: {str(e)}") raise - def _generate_sections(self, output_dir: str) -> None: + def _generate_sections(self, output_dir: str, static_dir: str) -> None: """ Generates Python files for each section in the report, including subsections and its components (plots, dataframes, markdown). @@ -316,6 +316,8 @@ def _generate_sections(self, output_dir: str) -> None: ---------- output_dir : str The folder where section files will be saved. + static_dir : str + The folder where the static files will be saved. """ self.report.logger.info("Starting to generate sections for the report.") @@ -342,7 +344,9 @@ def _generate_sections(self, output_dir: str) -> None: # Generate content and imports for the subsection subsection_content, subsection_imports = ( - self._generate_subsection(subsection) + self._generate_subsection( + subsection, static_dir=static_dir + ) ) # Flatten the subsection_imports into a single list @@ -379,7 +383,9 @@ def _generate_sections(self, output_dir: str) -> None: self.report.logger.error(f"Error generating sections: {str(e)}") raise - def _generate_subsection(self, subsection) -> tuple[List[str], List[str]]: + def _generate_subsection( + self, subsection, static_dir + ) -> tuple[List[str], List[str]]: """ Generate code to render components (plots, dataframes, markdown) in the given subsection, creating imports and content for the subsection based on the component type. @@ -388,6 +394,8 @@ def _generate_subsection(self, subsection) -> tuple[List[str], List[str]]: ---------- subsection : Subsection The subsection containing the components. + static_dir : str + The folder where the static files will be saved. Returns ------- @@ -416,7 +424,9 @@ def _generate_subsection(self, subsection) -> tuple[List[str], List[str]]: # Handle different types of components if component.component_type == r.ComponentType.PLOT: - subsection_content.extend(self._generate_plot_content(component)) + subsection_content.extend( + self._generate_plot_content(component, static_dir=static_dir) + ) elif component.component_type == r.ComponentType.DATAFRAME: subsection_content.extend(self._generate_dataframe_content(component)) # If md files is called "description.md", do not include it in the report @@ -445,9 +455,7 @@ def _generate_subsection(self, subsection) -> tuple[List[str], List[str]]: ) return subsection_content, subsection_imports - def _generate_plot_content( - self, plot, static_dir: str = STATIC_FILES_DIR - ) -> List[str]: + def _generate_plot_content(self, plot, static_dir: str) -> List[str]: """ Generate content for a plot component based on the plot type (static or interactive). @@ -460,8 +468,8 @@ def _generate_plot_content( ------- list : List[str] The list of content lines for the plot. - static_dir : str, optional - The folder where the static files will be saved (default is STATIC_FILES_DIR). + static_dir : str + The folder where the static files will be saved. """ plot_content = [] # Add title From a361d994f99f2f3a2de0de2114011c88651b7064 Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Sun, 9 Mar 2025 13:20:42 +0100 Subject: [PATCH 58/91] :bug: allow to specify log directory --- src/vuegen/__main__.py | 4 ++-- src/vuegen/config_manager.py | 4 +++- src/vuegen/report_generator.py | 10 +++++++++- src/vuegen/utils.py | 20 +++++++++++--------- 4 files changed, 25 insertions(+), 13 deletions(-) diff --git a/src/vuegen/__main__.py b/src/vuegen/__main__.py index 2f2565a..e911a59 100644 --- a/src/vuegen/__main__.py +++ b/src/vuegen/__main__.py @@ -36,8 +36,8 @@ def main(): logger_suffix = f"{report_type}_report_{str(report_name)}" # Initialize logger - logger = get_logger(f"{logger_suffix}") - + logger, logfile = get_logger(f"{logger_suffix}") + logger.info("logfile: %s", logfile) # Generate the report report_generator.get_report( report_type=report_type, diff --git a/src/vuegen/config_manager.py b/src/vuegen/config_manager.py index f671601..9ebab17 100644 --- a/src/vuegen/config_manager.py +++ b/src/vuegen/config_manager.py @@ -22,7 +22,9 @@ def __init__(self, logger: Optional[logging.Logger] = None): logger : logging.Logger, optional A logger instance for the class. If not provided, a default logger will be created. """ - self.logger = logger or get_logger("report") + if logger is None: + logger, _ = get_logger("report") + self.logger = logger def _create_title_fromdir(self, file_dirname: str) -> str: """ diff --git a/src/vuegen/report_generator.py b/src/vuegen/report_generator.py index e09900d..668adb2 100644 --- a/src/vuegen/report_generator.py +++ b/src/vuegen/report_generator.py @@ -15,6 +15,7 @@ def get_report( config_path: str = None, dir_path: str = None, streamlit_autorun: bool = False, + output_dir: Path = None, ) -> None: """ Generate and run a report based on the specified engine. @@ -37,9 +38,16 @@ def get_report( ValueError If neither 'config_path' nor 'directory' is provided. """ + if output_dir is None: + output_dir = Path(".") + else: + output_dir = Path(output_dir) # Initialize logger only if it's not provided if logger is None: - logger = get_logger("report") + _folder = "logs" + if output_dir: + _folder = output_dir / _folder + logger, _ = get_logger("report", folder=_folder) # Create the config manager object config_manager = ConfigManager(logger) diff --git a/src/vuegen/utils.py b/src/vuegen/utils.py index 41fa4e7..42c8c36 100644 --- a/src/vuegen/utils.py +++ b/src/vuegen/utils.py @@ -635,9 +635,11 @@ def generate_log_filename(folder: str = "logs", suffix: str = "") -> str: str The file path to the log file """ - # PRECONDITIONS - create_folder(folder) - + try: + # PRECONDITIONS + create_folder(folder) # ? Path(folder).mkdir(parents=True, exist_ok=True) + except OSError as e: + raise OSError(f"Error creating directory '{folder}': {e}") # MAIN FUNCTION log_filename = get_time(incl_timezone=False) + "_" + suffix + ".log" log_filepath = os.path.join(folder, log_filename) @@ -703,7 +705,7 @@ def init_log( return logger -def get_logger(log_suffix): +def get_logger(log_suffix, folder="logs", display=True) -> tuple[logging.Logger, str]: """ Initialize the logger with a log file name that includes an optional suffix. @@ -714,19 +716,19 @@ def get_logger(log_suffix): Returns ------- - logging.Logger - An initialized logger instance. + tuple[logging.Logger, str + A tuple containing the logger instance and the log file path. """ # Generate log file name - log_file = generate_log_filename(suffix=log_suffix) + log_file = generate_log_filename(folder=folder, suffix=log_suffix) # Initialize logger - logger = init_log(log_file, display=True) + logger = init_log(log_file, display=display) # Log the path to the log file logger.info(f"Path to log file: {log_file}") - return logger + return logger, log_file def print_completion_message(report_type: str): From 83b1b66e4ba3987e9d8f4802e873cba0204fe463 Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Sun, 9 Mar 2025 13:23:04 +0100 Subject: [PATCH 59/91] :sparkles: allow to set output directory, pass on to get_report - initialize logging directory at output directory --- gui/app.py | 70 ++++++++++++++++++++++++++++++---- src/vuegen/report_generator.py | 15 +++++--- 2 files changed, 72 insertions(+), 13 deletions(-) diff --git a/gui/app.py b/gui/app.py index fed9bb9..d7df6b2 100644 --- a/gui/app.py +++ b/gui/app.py @@ -28,16 +28,21 @@ # from vuegen.__main__ import main from vuegen.report import ReportType +from vuegen.utils import get_logger customtkinter.set_appearance_mode("system") customtkinter.set_default_color_theme("dark-blue") -from tkinter import filedialog +import traceback +from tkinter import filedialog, messagebox from vuegen import report_generator from vuegen.utils import print_completion_message -app_path = Path(__file__).absolute() +app_path = Path(__file__).absolute().resolve() print("app_path:", app_path) +output_dir = (Path.home() / "vuegen_gen" / "reports").resolve() +print("output_dir:", output_dir) +output_dir.mkdir(exist_ok=True, parents=True) ########################################################################################## # Path to example data dependend on how the GUI is run @@ -82,20 +87,45 @@ # return inner -def create_run_vuegen(is_dir, config_path, report_type, run_streamlit): +def create_run_vuegen( + is_dir, config_path, report_type, run_streamlit, output_dir_entry +): def inner(): kwargs = {} print(f"{is_dir.get() = }") if is_dir.get(): kwargs["dir_path"] = config_path.get() + report_name = Path(config_path.get()).stem else: kwargs["config_path"] = config_path.get() + report_name = Path(config_path.get()).stem kwargs["report_type"] = report_type.get() print(f"{run_streamlit.get() = }") kwargs["streamlit_autorun"] = run_streamlit.get() + kwargs["output_dir"] = output_dir_entry.get() print("kwargs:") pprint(kwargs) - report_generator.get_report(**kwargs) + try: + # Define logger suffix based on report type and name + logger_suffix = f"{report_type.get()}_report_{str(report_name)}" + + # Initialize logger + kwargs["logger"], log_file = get_logger( + f"{logger_suffix}", + folder=(Path(kwargs["output_dir"]) / "logs").as_posix(), + ) + report_generator.get_report(**kwargs) + messagebox.showinfo( + "Success", + "Report generation completed successfully." f"\nLogs at {log_file}", + ) + except Exception as e: + stacktrace = traceback.format_exc() + messagebox.showerror( + "Error", + f"An error occurred: {e}\n\n{stacktrace}" + f"\n See logs for more details {log_file}", + ) print_completion_message(report_type.get()) return inner @@ -115,7 +145,7 @@ def radio_button_callback(): def create_select_directory(string_var): def select_directory(): - directory = filedialog.askdirectory() + directory = filedialog.askdirectory(initialdir=string_var.get()) string_var.set(directory) return select_directory @@ -124,7 +154,7 @@ def select_directory(): ########################################################################################## # APP app = customtkinter.CTk() -app.geometry("600x400") +app.geometry("620x500") app.title("VueGen GUI") ########################################################################################## @@ -207,15 +237,39 @@ def select_directory(): ) ctk_radio_st_autorun_0.grid(row=5, column=1, padx=20, pady=20) +########################################################################################## +# output directory selection +# ctk_label_outdir = customtkinter.CTkLabel +output_dir_entry = tk.StringVar(value=str(output_dir)) +select_output_dir = create_select_directory(output_dir_entry) +select_output_dir_button = customtkinter.CTkButton( + app, text="Select Output Directory", command=select_output_dir +) +select_output_dir_button.grid(row=6, column=2, columnspan=2, padx=5, pady=10) + +ctk_entry_outpath = customtkinter.CTkEntry( + app, + width=400, + textvariable=output_dir_entry, +) +ctk_entry_outpath.grid(row=6, column=0, columnspan=2, padx=10, pady=10) +ctk_label_appath = customtkinter.CTkLabel( + app, + text=f"App path: {app_path}", +) +ctk_label_appath.grid(row=7, column=0, columnspan=2, padx=20, pady=5) + ########################################################################################## # Run VueGen button -run_vuegen = create_run_vuegen(is_dir, config_path, report_type, run_streamlit) +run_vuegen = create_run_vuegen( + is_dir, config_path, report_type, run_streamlit, output_dir_entry +) run_button = customtkinter.CTkButton( app, text="Run VueGen", command=run_vuegen, ) -run_button.grid(row=6, column=0, columnspan=2, padx=20, pady=20) +run_button.grid(row=8, column=0, columnspan=2, padx=20, pady=20) ########################################################################################## # Run the app in the mainloop diff --git a/src/vuegen/report_generator.py b/src/vuegen/report_generator.py index 668adb2..7764cf1 100644 --- a/src/vuegen/report_generator.py +++ b/src/vuegen/report_generator.py @@ -1,6 +1,7 @@ import logging import shutil import sys +from pathlib import Path from .config_manager import ConfigManager from .quarto_reportview import QuartoReportView @@ -68,12 +69,14 @@ def get_report( # Create and run ReportView object based on its type if report_type == ReportType.STREAMLIT: + base_dir = output_dir / "streamlit_report" + sections_dir = base_dir / "sections" + static_files_dir = base_dir / "static" st_report = StreamlitReportView( report=report, report_type=report_type, streamlit_autorun=streamlit_autorun ) - st_report.generate_report() - st_report.run_report() - + st_report.generate_report(output_dir=sections_dir, static_dir=static_files_dir) + st_report.run_report(output_dir=sections_dir) else: # Check if Quarto is installed if shutil.which("quarto") is None and not hasattr( @@ -85,6 +88,8 @@ def get_report( raise RuntimeError( "Quarto is not installed. Please install Quarto before generating this report type." ) + base_dir = output_dir / "quarto_report" + static_files_dir = base_dir / "static" quarto_report = QuartoReportView(report=report, report_type=report_type) - quarto_report.generate_report() - quarto_report.run_report() + quarto_report.generate_report(output_dir=base_dir, static_dir=static_files_dir) + quarto_report.run_report(output_dir=base_dir) From 283e09ab62a37fb4912a8b43c39cfe779404834a Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Mon, 10 Mar 2025 10:31:48 +0100 Subject: [PATCH 60/91] :art: separate by row, cont. increments --- gui/app.py | 45 +++++++++++++++++++++++++++------------------ 1 file changed, 27 insertions(+), 18 deletions(-) diff --git a/gui/app.py b/gui/app.py index d7df6b2..c24aa3b 100644 --- a/gui/app.py +++ b/gui/app.py @@ -157,13 +157,14 @@ def select_directory(): app.geometry("620x500") app.title("VueGen GUI") +row_count = 0 ########################################################################################## # Config or directory input ctk_label_config = customtkinter.CTkLabel( app, text="Add path to config file or directory. Select radio button accordingly", ) -ctk_label_config.grid(row=0, column=0, columnspan=2, padx=20, pady=20) +ctk_label_config.grid(row=row_count, column=0, columnspan=2, padx=20, pady=20) is_dir = tk.BooleanVar(value=True) callback_radio_config = create_radio_button_callback(is_dir, name="is_dir") ctk_radio_config_0 = customtkinter.CTkRadioButton( @@ -173,7 +174,9 @@ def select_directory(): variable=is_dir, value=False, ) -ctk_radio_config_0.grid(row=1, column=0, padx=20, pady=2) +row_count += 1 +########################################################################################## +ctk_radio_config_0.grid(row=row_count, column=0, padx=20, pady=2) ctk_radio_config_1 = customtkinter.CTkRadioButton( app, text="Use dir", @@ -181,7 +184,7 @@ def select_directory(): variable=is_dir, value=True, ) -ctk_radio_config_1.grid(row=1, column=1, padx=20, pady=2) +ctk_radio_config_1.grid(row=row_count, column=1, padx=20, pady=2) config_path = tk.StringVar(value=str(path_to_dat)) config_path_entry = customtkinter.CTkEntry( @@ -189,13 +192,15 @@ def select_directory(): width=400, textvariable=config_path, ) -config_path_entry.grid(row=2, column=0, columnspan=2, padx=5, pady=10) +row_count += 1 +########################################################################################## +config_path_entry.grid(row=row_count, column=0, columnspan=2, padx=5, pady=10) select_directory = create_select_directory(config_path) select_button = customtkinter.CTkButton( app, text="Select Directory", command=select_directory ) -select_button.grid(row=2, column=2, columnspan=2, padx=5, pady=10) - +select_button.grid(row=row_count, column=2, columnspan=2, padx=5, pady=10) +row_count += 1 ########################################################################################## # Report type dropdown # - get list of report types from Enum @@ -204,7 +209,9 @@ def select_directory(): app, text="Select type of report to generate (using only streamlit for now)", ) -ctk_label_report.grid(row=3, column=0, columnspan=2, padx=20, pady=20) +ctk_label_report.grid(row=row_count, column=0, columnspan=2, padx=20, pady=20) +row_count += 1 +########################################################################################## report_type = tk.StringVar(value=report_types[0]) report_dropdown = customtkinter.CTkOptionMenu( app, @@ -212,8 +219,8 @@ def select_directory(): variable=report_type, command=optionmenu_callback, ) -report_dropdown.grid(row=4, column=0, columnspan=2, padx=20, pady=20) - +report_dropdown.grid(row=row_count, column=0, columnspan=2, padx=20, pady=20) +row_count += 1 ########################################################################################## # Run Streamlit radio button run_streamlit = tk.BooleanVar(value=True) @@ -227,7 +234,7 @@ def select_directory(): variable=run_streamlit, command=callback_radio_st_run, ) -ctk_radio_st_autorun_1.grid(row=5, column=0, padx=20, pady=20) +ctk_radio_st_autorun_1.grid(row=row_count, column=0, padx=20, pady=20) ctk_radio_st_autorun_0 = customtkinter.CTkRadioButton( app, text="skip starting streamlit", @@ -235,8 +242,8 @@ def select_directory(): variable=run_streamlit, command=callback_radio_st_run, ) -ctk_radio_st_autorun_0.grid(row=5, column=1, padx=20, pady=20) - +ctk_radio_st_autorun_0.grid(row=row_count, column=1, padx=20, pady=20) +row_count += 1 ########################################################################################## # output directory selection # ctk_label_outdir = customtkinter.CTkLabel @@ -245,20 +252,22 @@ def select_directory(): select_output_dir_button = customtkinter.CTkButton( app, text="Select Output Directory", command=select_output_dir ) -select_output_dir_button.grid(row=6, column=2, columnspan=2, padx=5, pady=10) +select_output_dir_button.grid(row=row_count, column=2, columnspan=2, padx=5, pady=10) ctk_entry_outpath = customtkinter.CTkEntry( app, width=400, textvariable=output_dir_entry, ) -ctk_entry_outpath.grid(row=6, column=0, columnspan=2, padx=10, pady=10) +ctk_entry_outpath.grid(row=row_count, column=0, columnspan=2, padx=10, pady=10) +row_count += 1 +########################################################################################## ctk_label_appath = customtkinter.CTkLabel( app, text=f"App path: {app_path}", ) -ctk_label_appath.grid(row=7, column=0, columnspan=2, padx=20, pady=5) - +ctk_label_appath.grid(row=row_count, column=0, columnspan=2, padx=20, pady=5) +row_count += 1 ########################################################################################## # Run VueGen button run_vuegen = create_run_vuegen( @@ -269,8 +278,8 @@ def select_directory(): text="Run VueGen", command=run_vuegen, ) -run_button.grid(row=8, column=0, columnspan=2, padx=20, pady=20) - +run_button.grid(row=row_count, column=0, columnspan=2, padx=20, pady=20) +row_count += 1 ########################################################################################## # Run the app in the mainloop app.mainloop() From 0c83c9f91d90ba36a7a61911c1df4034eac8ae42 Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Mon, 10 Mar 2025 11:42:47 +0100 Subject: [PATCH 61/91] :bug: fix #89 - check if output was actually created --- gui/app.py | 1 + src/vuegen/quarto_reportview.py | 21 +++++++++++++-------- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/gui/app.py b/gui/app.py index c24aa3b..2aeb851 100644 --- a/gui/app.py +++ b/gui/app.py @@ -119,6 +119,7 @@ def inner(): "Success", "Report generation completed successfully." f"\nLogs at {log_file}", ) + print_completion_message(report_type.get()) except Exception as e: stacktrace = traceback.format_exc() messagebox.showerror( diff --git a/src/vuegen/quarto_reportview.py b/src/vuegen/quarto_reportview.py index bb17606..e9ba2f3 100644 --- a/src/vuegen/quarto_reportview.py +++ b/src/vuegen/quarto_reportview.py @@ -168,8 +168,8 @@ def run_report(self, output_dir: str = BASE_DIR) -> None: """ # from quarto_cli import run_quarto # entrypoint of quarto-cli not in module? - file_path_to_qmd = str(Path(output_dir) / f"{self.BASE_DIR}.qmd") - args = [self.quarto_path, "render", file_path_to_qmd] + file_path_to_qmd = Path(output_dir) / f"{self.BASE_DIR}.qmd" + args = [self.quarto_path, "render", str(file_path_to_qmd)] self.report.logger.info( f"Running '{self.report.title}' '{self.report_type}' report with {args!r}" ) @@ -191,8 +191,13 @@ def run_report(self, output_dir: str = BASE_DIR) -> None: args, check=True, ) + out_path = file_path_to_qmd.with_suffix(f".{self.report_type.lower()}") + if self.report_type in [r.ReportType.REVEALJS, r.ReportType.JUPYTER]: + out_path = file_path_to_qmd.with_suffix(".html") + if not out_path.exists(): + raise FileNotFoundError(f"Report file could not be created: {out_path}") if self.report_type == r.ReportType.JUPYTER: - args = [self.quarto_path, "convert", file_path_to_qmd] + args = [self.quarto_path, "convert", str(file_path_to_qmd)] subprocess.run( args, check=True, @@ -208,11 +213,11 @@ def run_report(self, output_dir: str = BASE_DIR) -> None: f"Error running '{self.report.title}' {self.report_type} report: {str(e)}" ) raise - except FileNotFoundError as e: - self.report.logger.error( - f"Quarto is not installed. Please install Quarto to run the report: {str(e)}" - ) - raise + # except FileNotFoundError as e: + # self.report.logger.error( + # f"Quarto is not installed. Please install Quarto to run the report: {str(e)}" + # ) + # raise def _create_yaml_header(self) -> str: """ From aaeadd472a9dc010ee7cd7cd17efabf5b7a9d40f Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Mon, 10 Mar 2025 20:59:31 +0100 Subject: [PATCH 62/91] =?UTF-8?q?=F0=9F=9A=A7=20allow=20to=20set=20Python?= =?UTF-8?q?=20Path=20manually?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - manipulate PATH and add logs --- gui/app.py | 87 ++++++++++++++++++++++++++++----- src/vuegen/quarto_reportview.py | 10 ++-- 2 files changed, 82 insertions(+), 15 deletions(-) diff --git a/gui/app.py b/gui/app.py index 2aeb851..b6bc286 100644 --- a/gui/app.py +++ b/gui/app.py @@ -19,6 +19,7 @@ report generation. """ +import os import sys import tkinter as tk from pathlib import Path @@ -43,7 +44,7 @@ output_dir = (Path.home() / "vuegen_gen" / "reports").resolve() print("output_dir:", output_dir) output_dir.mkdir(exist_ok=True, parents=True) - +_PATH = f'{os.environ["PATH"]}' ########################################################################################## # Path to example data dependend on how the GUI is run if getattr(sys, "frozen", False) and hasattr(sys, "_MEIPASS"): @@ -51,6 +52,10 @@ path_to_dat = ( app_path.parent / "example_data/Basic_example_vuegen_demo_notebook" ).resolve() + quarto_bin_path = os.path.join(sys._MEIPASS, "quarto_cli", "bin") + quarto_share_path = os.path.join(sys._MEIPASS, "quarto_cli", "share") + _PATH = os.pathsep.join([quarto_bin_path, quarto_share_path, _PATH]) + os.environ["PATH"] = _PATH elif app_path.parent.name == "gui": # should be always the case for GUI run from command line path_to_dat = ( @@ -63,9 +68,10 @@ else: path_to_dat = "docs/example_data/Basic_example_vuegen_demo_notebook" - +print(f"{_PATH = }") ########################################################################################## # callbacks +# using main entry point of vuegen # def create_run_vuegen(is_dir, config_path, report_type, run_streamlit): # def inner(): # args = ["vuegen"] @@ -88,7 +94,7 @@ def create_run_vuegen( - is_dir, config_path, report_type, run_streamlit, output_dir_entry + is_dir, config_path, report_type, run_streamlit, output_dir_entry, python_dir_entry ): def inner(): kwargs = {} @@ -105,7 +111,21 @@ def inner(): kwargs["output_dir"] = output_dir_entry.get() print("kwargs:") pprint(kwargs) + # os.environ["PYTHONPATH"] = os.pathsep.join(sys.path) + if python_dir_entry.get(): + # os.environ["PYTHONHOME"] = python_dir_entry.get() + os.environ["PATH"] = python_dir_entry.get() + os.pathsep + _PATH + if getattr(sys, "frozen", False) and hasattr(sys, "_MEIPASS"): + os.environ["PATH"] = os.pathsep.join( + [ + quarto_bin_path, + quarto_share_path, + python_dir_entry.get(), + _PATH, + ] + ) try: + os.chdir(kwargs["output_dir"]) # Change the working directory # Define logger suffix based on report type and name logger_suffix = f"{report_type.get()}_report_{str(report_name)}" @@ -114,6 +134,9 @@ def inner(): f"{logger_suffix}", folder=(Path(kwargs["output_dir"]) / "logs").as_posix(), ) + kwargs["logger"].info("logfile: %s", log_file) + kwargs["logger"].debug("sys.path: %s", sys.path) + kwargs["logger"].debug("PATH (in app): %s ", os.environ["PATH"]) report_generator.get_report(**kwargs) messagebox.showinfo( "Success", @@ -127,7 +150,6 @@ def inner(): f"An error occurred: {e}\n\n{stacktrace}" f"\n See logs for more details {log_file}", ) - print_completion_message(report_type.get()) return inner @@ -155,7 +177,7 @@ def select_directory(): ########################################################################################## # APP app = customtkinter.CTk() -app.geometry("620x500") +app.geometry("620x600") app.title("VueGen GUI") row_count = 0 @@ -206,6 +228,7 @@ def select_directory(): # Report type dropdown # - get list of report types from Enum report_types = [report_type.value.lower() for report_type in ReportType] +# report_types = report_types[:2] # only streamlit and html for now ctk_label_report = customtkinter.CTkLabel( app, text="Select type of report to generate (using only streamlit for now)", @@ -247,14 +270,16 @@ def select_directory(): row_count += 1 ########################################################################################## # output directory selection -# ctk_label_outdir = customtkinter.CTkLabel +ctk_label_outdir = customtkinter.CTkLabel(app, text="Select output directory:") +ctk_label_outdir.grid(row=row_count, column=0, columnspan=1, padx=10, pady=5) +row_count += 1 +########################################################################################## output_dir_entry = tk.StringVar(value=str(output_dir)) select_output_dir = create_select_directory(output_dir_entry) select_output_dir_button = customtkinter.CTkButton( app, text="Select Output Directory", command=select_output_dir ) -select_output_dir_button.grid(row=row_count, column=2, columnspan=2, padx=5, pady=10) - +select_output_dir_button.grid(row=row_count, column=2, columnspan=1, padx=5, pady=10) ctk_entry_outpath = customtkinter.CTkEntry( app, width=400, @@ -263,16 +288,54 @@ def select_directory(): ctk_entry_outpath.grid(row=row_count, column=0, columnspan=2, padx=10, pady=10) row_count += 1 ########################################################################################## -ctk_label_appath = customtkinter.CTkLabel( +# Python binary selection +# ctk_label_python = customtkinter.CTkLabel +ctk_label_outdir = customtkinter.CTkLabel(app, text="Select Python binary:") +ctk_label_outdir.grid(row=row_count, column=0, columnspan=1, padx=10, pady=5) +row_count += 1 +########################################################################################## +python_dir_entry = tk.StringVar(value="") +select_python_bin = create_select_directory(python_dir_entry) +select_python_bin_button = customtkinter.CTkButton( + app, text="Select Python binary", command=select_python_bin +) +select_python_bin_button.grid(row=row_count, column=2, columnspan=1, padx=5, pady=5) + +ctk_entry_python = customtkinter.CTkEntry( + app, + width=400, + textvariable=python_dir_entry, +) +ctk_entry_python.grid(row=row_count, column=0, columnspan=2, padx=10, pady=5) +row_count += 1 +########################################################################################## +ctk_label_env_path = customtkinter.CTkLabel(app, text="PATH:") +ctk_label_env_path.grid(row=row_count, column=0, columnspan=1, padx=2, pady=5) +env_path = tk.StringVar(value=os.environ.get("PATH", "not found")) +ctk_entry_path_env = customtkinter.CTkEntry( app, - text=f"App path: {app_path}", + width=400, + textvariable=env_path, ) -ctk_label_appath.grid(row=row_count, column=0, columnspan=2, padx=20, pady=5) +ctk_entry_path_env.grid(row=row_count, column=1, columnspan=2, padx=10, pady=5) row_count += 1 ########################################################################################## +# ctk_label_appath = customtkinter.CTkLabel( +# app, +# text=f"App path: {app_path}", +# wraplength=600, +# ) +# ctk_label_appath.grid(row=row_count, column=0, columnspan=3, padx=10, pady=5) +# row_count += 1 +########################################################################################## # Run VueGen button run_vuegen = create_run_vuegen( - is_dir, config_path, report_type, run_streamlit, output_dir_entry + is_dir=is_dir, + config_path=config_path, + report_type=report_type, + run_streamlit=run_streamlit, + output_dir_entry=output_dir_entry, + python_dir_entry=python_dir_entry, ) run_button = customtkinter.CTkButton( app, diff --git a/src/vuegen/quarto_reportview.py b/src/vuegen/quarto_reportview.py index e9ba2f3..50ddbea 100644 --- a/src/vuegen/quarto_reportview.py +++ b/src/vuegen/quarto_reportview.py @@ -24,14 +24,18 @@ def __init__(self, report: r.Report, report_type: r.ReportType): super().__init__(report=report, report_type=report_type) self.BUNDLED_EXECUTION = False self.quarto_path = "quarto" + # self.env_vars = os.environ.copy() if getattr(sys, "frozen", False) and hasattr(sys, "_MEIPASS"): self.report.logger.info("running in a PyInstaller bundle") self.BUNDLED_EXECUTION = True self.report.logger.debug(f"sys._MEIPASS: {sys._MEIPASS}") - self.quarto_path = str(Path(sys._MEIPASS) / "quarto_cli" / "bin" / "quarto") else: self.report.logger.info("running in a normal Python process") + self.report.logger.debug("env_vars (QuartoReport): %s", os.environ) + self.report.logger.debug(f"PATH: {os.environ['PATH']}") + self.report.logger.debug(f"sys.path: {sys.path}") + def generate_report( self, output_dir: Path = BASE_DIR, static_dir: Path = STATIC_FILES_DIR ) -> None: @@ -179,11 +183,11 @@ def run_report(self, output_dir: str = BASE_DIR) -> None: r.ReportType.ODT, ]: subprocess.run( - [self.quarto_path, "install", "tinytex"], + [self.quarto_path, "install", "tinytex", "--no-prompt"], check=True, ) subprocess.run( - [self.quarto_path, "install", "chromium"], + [self.quarto_path, "install", "chromium", "--no-prompt"], check=True, ) try: From 9682be698b133a2cb0a0b73d667a02d48c8c53ed Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Mon, 10 Mar 2025 21:00:27 +0100 Subject: [PATCH 63/91] :bug: set generic kernel name - otherwise it uses local defaults on device --- src/vuegen/quarto_reportview.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/vuegen/quarto_reportview.py b/src/vuegen/quarto_reportview.py index 50ddbea..558cc08 100644 --- a/src/vuegen/quarto_reportview.py +++ b/src/vuegen/quarto_reportview.py @@ -239,6 +239,7 @@ def _create_yaml_header(self) -> str: execute: echo: false output: asis +jupyter: python3 format:""" # Define format-specific YAML configurations From 1cefe8c343ad63f6b67336890e078861fd5ff718 Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Mon, 10 Mar 2025 21:00:45 +0100 Subject: [PATCH 64/91] :art: fix typo in function name --- src/vuegen/quarto_reportview.py | 2 +- src/vuegen/report.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/vuegen/quarto_reportview.py b/src/vuegen/quarto_reportview.py index 558cc08..94a606d 100644 --- a/src/vuegen/quarto_reportview.py +++ b/src/vuegen/quarto_reportview.py @@ -512,7 +512,7 @@ def _generate_plot_content( # Add code to generate network depending on the report type if is_report_static: - plot.save_netwrok_image(networkx_graph, static_plot_path, "png") + plot.save_network_image(networkx_graph, static_plot_path, "png") plot_content.append(self._generate_image_content(static_plot_path)) else: plot_content.append(self._generate_plot_code(plot, html_plot_file)) diff --git a/src/vuegen/report.py b/src/vuegen/report.py index be2315d..68312aa 100644 --- a/src/vuegen/report.py +++ b/src/vuegen/report.py @@ -277,7 +277,7 @@ def read_network(self) -> nx.Graph: f"An error occurred while reading the network file: {str(e)}" ) - def save_netwrok_image( + def save_network_image( self, G: nx.Graph, output_file: str, format: str, dpi: int = 300 ) -> None: """ @@ -294,6 +294,7 @@ def save_netwrok_image( dpi : int, optional The resolution of the image in dots per inch (default is 300). """ + self.logger.debug("Try to save network as PyVis network: %s.", output_file) # Check if the output file path is valid if not os.path.isdir(os.path.dirname(output_file)): self.logger.error( @@ -339,6 +340,7 @@ def create_and_save_pyvis_network(self, G: nx.Graph, output_file: str) -> Networ net : pyvis.network.Network A PyVis network object. """ + self.logger.debug("Try to save network as PyVis network: %s.", output_file) # Check if the network object and output file path are valid if not isinstance(G, nx.Graph): self.logger.error( From f6cbac97146aaa542be3b8a923a365057334a639 Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Tue, 11 Mar 2025 12:54:32 +0100 Subject: [PATCH 65/91] :art: use isostandard for log file format - but relace colons with hypens in hour:minutes:seconds --- src/vuegen/utils.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/vuegen/utils.py b/src/vuegen/utils.py index 42c8c36..9404251 100644 --- a/src/vuegen/utils.py +++ b/src/vuegen/utils.py @@ -596,27 +596,27 @@ def get_time(incl_time: bool = True, incl_timezone: bool = True) -> str: y = str(the_time.year) M = str(the_time.month) d = str(the_time.day) - h = str(the_time.hour) - m = str(the_time.minute) - s = str(the_time.second) # putting date parts into one string if incl_time and incl_timezone: - fname = "_".join([y + M + d, h + m + s, timezone]) + fname = the_time.isoformat(sep="_", timespec="seconds") + "_" + timezone elif incl_time: - fname = "_".join([y + M + d, h + m + s]) + fname = the_time.isoformat(sep="_", timespec="seconds") elif incl_timezone: - fname = "_".join([y + M + d, timezone]) + fname = "_".join([the_time.isoformat(sep="_", timespec="hours")[:-3], timezone]) else: fname = y + M + d + # optional + fname = fname.replace(":", "-") # remove ':' from hours, minutes, seconds # POSTCONDITIONALS - parts = fname.split("_") - if incl_time and incl_timezone: - assert len(parts) == 3, f"time and/or timezone inclusion issue: {fname}" - elif incl_time or incl_timezone: - assert len(parts) == 2, f"time/timezone inclusion issue: {fname}" - else: - assert len(parts) == 1, f"time/timezone inclusion issue: {fname}" + # ! to delete (was it jused for something?) + # parts = fname.split("_") + # if incl_time and incl_timezone: + # assert len(parts) == 3, f"time and/or timezone inclusion issue: {fname}" + # elif incl_time or incl_timezone: + # assert len(parts) == 2, f"time/timezone inclusion issue: {fname}" + # else: + # assert len(parts) == 1, f"time/timezone inclusion issue: {fname}" return fname @@ -641,7 +641,7 @@ def generate_log_filename(folder: str = "logs", suffix: str = "") -> str: except OSError as e: raise OSError(f"Error creating directory '{folder}': {e}") # MAIN FUNCTION - log_filename = get_time(incl_timezone=False) + "_" + suffix + ".log" + log_filename = get_time(incl_timezone=True) + "_" + suffix + ".log" log_filepath = os.path.join(folder, log_filename) return log_filepath From 2d585f21325aad368a9e5d0f0ecb0de0500434fe Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Tue, 11 Mar 2025 13:58:40 +0100 Subject: [PATCH 66/91] :sparkles: this let's the interpreter look for the packages in the shipped folder - but they need to be present as full modules (which means that `.py` and not `.so` files are present? - will be large... but that is a side note Use Python=3.12 where pyvis examples should work in streamlit --- gui/README.md | 9 ++------- gui/app.py | 16 +++++++++++++--- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/gui/README.md b/gui/README.md index d9acc47..5642ebc 100644 --- a/gui/README.md +++ b/gui/README.md @@ -63,12 +63,7 @@ Windows and macOS specific options: ## Quarto notebook execution -- add python exe to bundle as suggested [on stackoverflow](https://stackoverflow.com/a/72639099/9684872) -- use [copy_python_executable.py](copy_python_executable.py) to copy the python executable to the bundle after PyInstaller is done +- add python exe to bundle as suggested [on stackoverflow](https://stackoverflow.com/a/72639099/9684872) [not this way at least] -Basic workflow for bundle: +## test shipping a python virtual environment with vuegen installed -1. use quarto (pandoc?) to convert qmd to ipynb -1. use nbconvert programmatically (see [docs](https://nbconvert.readthedocs.io/en/latest/execute_api.html#example)) - and the bundled Python executable to execute notebook -1. use quarto (pandoc?) ot convert executed ipynb to desired format diff --git a/gui/app.py b/gui/app.py index b6bc286..becec7e 100644 --- a/gui/app.py +++ b/gui/app.py @@ -56,6 +56,14 @@ quarto_share_path = os.path.join(sys._MEIPASS, "quarto_cli", "share") _PATH = os.pathsep.join([quarto_bin_path, quarto_share_path, _PATH]) os.environ["PATH"] = _PATH + # This requires that the python version is the same as the one used to create the executable + # in the Python environment the kernel is started from for quarto based reports + # os.environ["PYTHONPATH"] = os.pathsep.join( + # sys.path + # ) # ! requires kernel env with same Python env, but does not really seem to help + os.environ["PYTHONPATH"] = os.pathsep.join( + [sys.path[0], sys._MEIPASS] + ) # does not work elif app_path.parent.name == "gui": # should be always the case for GUI run from command line path_to_dat = ( @@ -111,10 +119,8 @@ def inner(): kwargs["output_dir"] = output_dir_entry.get() print("kwargs:") pprint(kwargs) - # os.environ["PYTHONPATH"] = os.pathsep.join(sys.path) + if python_dir_entry.get(): - # os.environ["PYTHONHOME"] = python_dir_entry.get() - os.environ["PATH"] = python_dir_entry.get() + os.pathsep + _PATH if getattr(sys, "frozen", False) and hasattr(sys, "_MEIPASS"): os.environ["PATH"] = os.pathsep.join( [ @@ -124,6 +130,10 @@ def inner(): _PATH, ] ) + else: + messagebox.showwarning( + "warning", "Running locally. Ignoring set Python Path" + ) try: os.chdir(kwargs["output_dir"]) # Change the working directory # Define logger suffix based on report type and name From 51cc2f045785554569573436c0714d250ed18d0d Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Tue, 11 Mar 2025 15:20:21 +0100 Subject: [PATCH 67/91] :bug: only add folder as import, not the zip file --- .github/workflows/cdci.yml | 5 +++-- gui/app.py | 7 +++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/cdci.yml b/.github/workflows/cdci.yml index 728e223..6e524f5 100644 --- a/.github/workflows/cdci.yml +++ b/.github/workflows/cdci.yml @@ -131,7 +131,7 @@ jobs: # - other-reports strategy: matrix: - python-version: ["3.11"] + python-version: ["3.12"] os: # https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/running-variations-of-jobs-in-a-workflow#example-using-a-multi-dimension-matrix # - runner: "ubuntu-latest" @@ -157,8 +157,9 @@ jobs: - name: Build executable run: | cd gui - pyinstaller -n vuegen_gui -D --collect-all pyvis --collect-all streamlit --collect-all st_aggrid --collect-all customtkinter --collect-all quarto_cli --add-data ../docs/example_data/Basic_example_vuegen_demo_notebook:example_data/Basic_example_vuegen_demo_notebook app.py + pyinstaller -n vuegen_gui -D --collect-all pyvis --collect-all streamlit --collect-all st_aggrid --collect-all customtkinter --collect-all quarto_cli --collect-all plotly --collect-all _plotly_utils --collect-all pyvis --collect-all pandas --collect-all numpy --collect-all matplotlib --collect-all openpyxl --collect-all xlrd --collect-all nbformat --collect-all nbclient --collect-all altair --collect-all itables --collect-all kaleido --collect-all vl_convert --collect-all yaml --collect-all dataframe_image --collect-all strenum --collect-all jinja2 --collect-all fastjsonschema --collect-all jsonschema --collect-all jsonschema_specifications --collect-all traitlets --collect-all referencing add-data ../docs/example_data/Basic_example_vuegen_demo_notebook:example_data/Basic_example_vuegen_demo_notebook app.py # -w -F + # replace by spec file once done... - name: Upload executable uses: actions/upload-artifact@v4 with: diff --git a/gui/app.py b/gui/app.py index becec7e..f2b0f01 100644 --- a/gui/app.py +++ b/gui/app.py @@ -61,9 +61,8 @@ # os.environ["PYTHONPATH"] = os.pathsep.join( # sys.path # ) # ! requires kernel env with same Python env, but does not really seem to help - os.environ["PYTHONPATH"] = os.pathsep.join( - [sys.path[0], sys._MEIPASS] - ) # does not work + os.environ["PYTHONPATH"] = sys._MEIPASS + # ([[sys.path[0], sys._MEIPASS]) # does not work when built on GitHub Actions elif app_path.parent.name == "gui": # should be always the case for GUI run from command line path_to_dat = ( @@ -321,7 +320,7 @@ def select_directory(): ########################################################################################## ctk_label_env_path = customtkinter.CTkLabel(app, text="PATH:") ctk_label_env_path.grid(row=row_count, column=0, columnspan=1, padx=2, pady=5) -env_path = tk.StringVar(value=os.environ.get("PATH", "not found")) +env_path = tk.StringVar(value=_PATH) ctk_entry_path_env = customtkinter.CTkEntry( app, width=400, From 038909d86f9b8d3ce63cf170be59065121ed8563 Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Tue, 11 Mar 2025 15:56:10 +0100 Subject: [PATCH 68/91] :bug: fix cdci --- .github/workflows/cdci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cdci.yml b/.github/workflows/cdci.yml index 6e524f5..a510000 100644 --- a/.github/workflows/cdci.yml +++ b/.github/workflows/cdci.yml @@ -157,7 +157,7 @@ jobs: - name: Build executable run: | cd gui - pyinstaller -n vuegen_gui -D --collect-all pyvis --collect-all streamlit --collect-all st_aggrid --collect-all customtkinter --collect-all quarto_cli --collect-all plotly --collect-all _plotly_utils --collect-all pyvis --collect-all pandas --collect-all numpy --collect-all matplotlib --collect-all openpyxl --collect-all xlrd --collect-all nbformat --collect-all nbclient --collect-all altair --collect-all itables --collect-all kaleido --collect-all vl_convert --collect-all yaml --collect-all dataframe_image --collect-all strenum --collect-all jinja2 --collect-all fastjsonschema --collect-all jsonschema --collect-all jsonschema_specifications --collect-all traitlets --collect-all referencing add-data ../docs/example_data/Basic_example_vuegen_demo_notebook:example_data/Basic_example_vuegen_demo_notebook app.py + pyinstaller -n vuegen_gui -D --collect-all pyvis --collect-all streamlit --collect-all st_aggrid --collect-all customtkinter --collect-all quarto_cli --collect-all plotly --collect-all _plotly_utils --collect-all pyvis --collect-all pandas --collect-all numpy --collect-all matplotlib --collect-all openpyxl --collect-all xlrd --collect-all nbformat --collect-all nbclient --collect-all altair --collect-all itables --collect-all kaleido --collect-all vl_convert --collect-all yaml --collect-all dataframe_image --collect-all strenum --collect-all jinja2 --collect-all fastjsonschema --collect-all jsonschema --collect-all jsonschema_specifications --collect-all traitlets --collect-all referencing --collect-all rpds --collect-all tenacity --add-data ../docs/example_data/Basic_example_vuegen_demo_notebook:example_data/Basic_example_vuegen_demo_notebook app.py # -w -F # replace by spec file once done... - name: Upload executable From 550c78a918cd484e774f39acb94b04957a4f32df Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Tue, 11 Mar 2025 16:34:38 +0100 Subject: [PATCH 69/91] =?UTF-8?q?=F0=9F=9A=A7=20build=20for=20python=203.1?= =?UTF-8?q?2=20with=20jupyter=20installed?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - conda base 3.12 with jupyter installed needed to execute - other than that 'only' vuegen required packges --- .github/workflows/cdci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cdci.yml b/.github/workflows/cdci.yml index a510000..5f691a4 100644 --- a/.github/workflows/cdci.yml +++ b/.github/workflows/cdci.yml @@ -157,8 +157,9 @@ jobs: - name: Build executable run: | cd gui - pyinstaller -n vuegen_gui -D --collect-all pyvis --collect-all streamlit --collect-all st_aggrid --collect-all customtkinter --collect-all quarto_cli --collect-all plotly --collect-all _plotly_utils --collect-all pyvis --collect-all pandas --collect-all numpy --collect-all matplotlib --collect-all openpyxl --collect-all xlrd --collect-all nbformat --collect-all nbclient --collect-all altair --collect-all itables --collect-all kaleido --collect-all vl_convert --collect-all yaml --collect-all dataframe_image --collect-all strenum --collect-all jinja2 --collect-all fastjsonschema --collect-all jsonschema --collect-all jsonschema_specifications --collect-all traitlets --collect-all referencing --collect-all rpds --collect-all tenacity --add-data ../docs/example_data/Basic_example_vuegen_demo_notebook:example_data/Basic_example_vuegen_demo_notebook app.py + pyinstaller -n vuegen_gui -D --collect-all pyvis --collect-all streamlit --collect-all st_aggrid --collect-all customtkinter --collect-all quarto_cli --collect-all plotly --collect-all _plotly_utils --collect-all pyvis --collect-all pandas --collect-all numpy --collect-all matplotlib --collect-all openpyxl --collect-all xlrd --collect-all nbformat --collect-all nbclient --collect-all altair --collect-all itables --collect-all kaleido --collect-all pyarrow --add-data ../docs/example_data/Basic_example_vuegen_demo_notebook:example_data/Basic_example_vuegen_demo_notebook app.py # -w -F + # --collect-all vl_convert --collect-all yaml --collect-all dataframe_image --collect-all strenum --collect-all jinja2 --collect-all fastjsonschema --collect-all jsonschema --collect-all jsonschema_specifications --collect-all traitlets --collect-all referencing --collect-all rpds --collect-all tenacity # replace by spec file once done... - name: Upload executable uses: actions/upload-artifact@v4 From 6bea275d94d8413dc72eaac82ab35b87761e4204 Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Wed, 12 Mar 2025 13:40:57 +0100 Subject: [PATCH 70/91] =?UTF-8?q?=F0=9F=9A=A7=20add=20makefile=20to=20docu?= =?UTF-8?q?ment=20local=20build=20on=20mac?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Makefile also has environment packages listed which could be added (be aware to use package folder name, not PyPI name, e.g. PIL and not pillow) - should work with local python environment with jupyter installed - switched to windowed build --- .github/workflows/cdci.yml | 4 +- gui/Makefile | 218 +++++++++++++++++++++++++++++++++++++ gui/README.md | 37 +++++++ 3 files changed, 257 insertions(+), 2 deletions(-) create mode 100644 gui/Makefile diff --git a/.github/workflows/cdci.yml b/.github/workflows/cdci.yml index 5f691a4..17b0cd9 100644 --- a/.github/workflows/cdci.yml +++ b/.github/workflows/cdci.yml @@ -157,9 +157,9 @@ jobs: - name: Build executable run: | cd gui - pyinstaller -n vuegen_gui -D --collect-all pyvis --collect-all streamlit --collect-all st_aggrid --collect-all customtkinter --collect-all quarto_cli --collect-all plotly --collect-all _plotly_utils --collect-all pyvis --collect-all pandas --collect-all numpy --collect-all matplotlib --collect-all openpyxl --collect-all xlrd --collect-all nbformat --collect-all nbclient --collect-all altair --collect-all itables --collect-all kaleido --collect-all pyarrow --add-data ../docs/example_data/Basic_example_vuegen_demo_notebook:example_data/Basic_example_vuegen_demo_notebook app.py + pyinstaller -n vuegen_gui -D -w --collect-all pyvis --collect-all streamlit --collect-all st_aggrid --collect-all customtkinter --collect-all quarto_cli --collect-all plotly --collect-all _plotly_utils --collect-all traitlets --collect-all referencing --collect-all rpds --collect-all tenacity --collect-all pyvis --collect-all pandas --collect-all numpy --collect-all matplotlib --collect-all openpyxl --collect-all xlrd --collect-all nbformat --collect-all nbclient --collect-all altair --collect-all itables --collect-all kaleido --collect-all pyarrow --collect-all dataframe_image --collect-all narwhals --collect-all PIL --add-data ../docs/example_data/Basic_example_vuegen_demo_notebook:example_data/Basic_example_vuegen_demo_notebook app.py # -w -F - # --collect-all vl_convert --collect-all yaml --collect-all dataframe_image --collect-all strenum --collect-all jinja2 --collect-all fastjsonschema --collect-all jsonschema --collect-all jsonschema_specifications --collect-all traitlets --collect-all referencing --collect-all rpds --collect-all tenacity + # --collect-all vl_convert --collect-all yaml --collect-all strenum --collect-all jinja2 --collect-all fastjsonschema --collect-all jsonschema --collect-all jsonschema_specifications # replace by spec file once done... - name: Upload executable uses: actions/upload-artifact@v4 diff --git a/gui/Makefile b/gui/Makefile new file mode 100644 index 0000000..73b01ba --- /dev/null +++ b/gui/Makefile @@ -0,0 +1,218 @@ +clean: + rm -r dist build lib vuegen.spec quarto_report streamlit_report + +clean-all: clean + rm -r logs + +bundle: + pip install -e ../. + pyinstaller \ + -n vuegen_gui \ + --noconfirm \ + --onedir \ + --windowed \ + --collect-all streamlit \ + --collect-all st_aggrid \ + --collect-all customtkinter \ + --collect-all quarto_cli \ + --collect-all plotly \ + --collect-all _plotly_utils \ + --collect-all pyvis \ + --collect-all pandas \ + --collect-all numpy \ + --collect-all matplotlib \ + --collect-all openpyxl \ + --collect-all xlrd \ + --collect-all nbformat \ + --collect-all nbclient \ + --collect-all altair \ + --collect-all itables \ + --collect-all kaleido \ + --collect-all pyarrow \ + --collect-all dataframe_image \ + --collect-all narwhals \ + --collect-all PIL \ + --collect-all traitlets \ + --collect-all referencing \ + --collect-all rpds \ + --collect-all tenacity \ + --add-data ../docs/example_data/Basic_example_vuegen_demo_notebook:example_data/Basic_example_vuegen_demo_notebook \ + app.py + + +# jupyter kernel specific: +# --collect-all vl_convert \ +# --collect-all yaml \ +# --collect-all strenum \ +# --collect-all jinja2 \ +# --collect-all fastjsonschema \ +# --collect-all jsonschema \ +# --collect-all jsonschema_specifications \ +# --collect-all nbclient \ +# --collect-all nbformat \ + +# beautifulsoup4, bleach, defusedxml, importlib-metadata, jinja2, jupyter-core, jupyterlab-pygments, markupsafe, mistune, nbclient, nbformat, packaging, pandocfilters, pygments, traitlets +# remaining packages in full environment: +# --collect-all jupyterlab \ +# --collect-all jupyter_core \ +# --collect-all yaml \ +# --collect-all ipykernel \ +# --collect-all nbconvert \ +# --collect-all notebook \ +# --collect-all ipywidgets \ +# --collect-all jupyter_console \ +# --collect-all jupyter_client \ +# --collect-all webencodings \ +# --collect-all wcwidth \ +# --collect-all pytz \ +# --collect-all python-decouple\ +# --collect-all pure-eval \ +# --collect-all ptyprocess \ +# --collect-all kaleido \ +# --collect-all fastjsonschema\ +# --collect-all xlrd \ +# --collect-all widgetsnbextension\ +# --collect-all wheel \ +# --collect-all websocket-client\ +# --collect-all webcolors \ +# --collect-all vl-convert-python\ +# --collect-all urllib3 \ +# --collect-all uri-template \ +# --collect-all tzdata \ +# --collect-all typing-extensions\ +# --collect-all types-python-dateutil \ +# --collect-all traitlets \ +# --collect-all tornado \ +# --collect-all toml \ +# --collect-all tinycss2 \ + +# --collect-all soupsieve \ +# --collect-all sniffio \ +# --collect-all smmap \ +# --collect-all six \ +# --collect-all setuptools \ +# --collect-all send2trash \ +# --collect-all rpds-py \ +# --collect-all rfc3986-validator\ +# --collect-all pyzmq \ +# --collect-all pyyaml \ +# --collect-all python-json-logger\ +# --collect-all pyparsing \ +# --collect-all pygments \ +# --collect-all pycparser \ +# --collect-all pyarrow \ +# --collect-all psutil \ +# --collect-all protobuf \ +# --collect-all propcache \ +# --collect-all prompt_toolkit\ +# --collect-all prometheus-client \ +# --collect-all platformdirs \ +# --collect-all pillow \ # PIL +# --collect-all pexpect \ +# --collect-all parso \ +# --collect-all pandocfilters\ +# --collect-all packaging \ +# --collect-all overrides \ +# --collect-all numpy \ +# --collect-all networkx \ +# --collect-all nest-asyncio \ +# --collect-all multidict \ +# --collect-all more-itertools\ +# --collect-all mistune \ +# --collect-all mdurl \ +# --collect-all MarkupSafe \ +# --collect-all lxml \ +# --collect-all kiwisolver \ +# --collect-all jupyterlab_widgets\ +# --collect-all jupyterlab_pygments \ +# --collect-all jsonpointer \ +# --collect-all jsonpickle \ +# --collect-all json5 \ +# --collect-all idna \ +# --collect-all h11\ +# --collect-all greenlet \ +# --collect-all frozenlist \ +# --collect-all fqdn \ +# --collect-all fonttools \ +# --collect-all executing \ +# --collect-all et-xmlfile \ +# --collect-all defusedxml \ +# --collect-all decorator \ +# --collect-all debugpy \ +# --collect-all cycler \ +# --collect-all cssselect \ +# --collect-all click \ +# --collect-all charset-normalizer\ +# --collect-all certifi \ +# --collect-all cachetools \ +# --collect-all blinker \ +# --collect-all bleach \ +# --collect-all babel \ +# --collect-all attrs \ +# --collect-all async-lru \ +# --collect-all asttokens \ +# --collect-all appnope \ +# --collect-all aiohappyeyeballs\ +# --collect-all yarl \ +# --collect-all terminado \ +# --collect-all stack_data \ +# --collect-all rfc3339-validator\ +# --collect-all requests \ +# --collect-all referencing\ +# --collect-all python-dateutil \ +# --collect-all pyee \ +# --collect-all plotly \ +# --collect-all openpyxl \ +# --collect-all matplotlib-inline\ +# --collect-all markdown-it-py \ +# --collect-all jupyter-core \ +# --collect-all jinja2 \ +# --collect-all jedi \ +# --collect-all ipython-pygments-lexers\ +# --collect-all httpcore \ +# --collect-all gitdb \ +# --collect-all cssutils \ +# --collect-all contourpy \ +# --collect-all comm \ +# --collect-all cffi \ +# --collect-all beautifulsoup4\ +# --collect-all anyio \ +# --collect-all aiosignal \ +# --collect-all rich \ +# --collect-all pydeck \ +# --collect-all playwright \ +# --collect-all pandas \ +# --collect-all matplotlib \ +# --collect-all jupyter-server-terminals\ +# --collect-all jupyter-client \ +# --collect-all jsonschema-specifications \ +# --collect-all ipython \ +# --collect-all httpx \ +# --collect-all gitpython \ +# --collect-all arrow \ +# --collect-all argon2-cffi-bindings\ +# --collect-all aiohttp \ +# --collect-all pyvis \ +# --collect-all jsonschema \ +# --collect-all isoduration\ +# --collect-all ipywidgets \ +# --collect-all ipykernel \ +# --collect-all argon2-cffi\ +# --collect-all nbformat \ +# --collect-all jupyter-console\ +# --collect-all altair \ +# --collect-all streamlit \ +# --collect-all nbclient \ +# --collect-all jupyter-events\ +# --collect-all streamlit-aggrid \ +# --collect-all nbconvert \ +# --collect-all jupyter-server\ +# --collect-all dataframe-image \ +# --collect-all notebook-shim \ +# --collect-all jupyterlab-server \ +# --collect-all jupyter-lsp \ +# --collect-all jupyterlab \ +# --collect-all notebook \ +# --collect-all jupyter \ +# --collect-all vuegen + diff --git a/gui/README.md b/gui/README.md index 5642ebc..3456a69 100644 --- a/gui/README.md +++ b/gui/README.md @@ -67,3 +67,40 @@ Windows and macOS specific options: ## test shipping a python virtual environment with vuegen installed +## Bundled PyInstaller execution (current status) + +1. Can be executed. Streamlit apps can be run (although sometimes not easily terminated) +2. All quarto based reports need to specify a path to a python environment where python 3.12 + is installed along `jupyter` + - This could be partly replace by a full anaconda distribution on the system. + - maybe a self-contained minimal virtual environment for kernel starting can be added later + - we could add some logic to make sure a correct path is added. + +### Create environment using conda + +```bash +conda create -n vuegen_gui -c conda-forge python=3.12 jupyter +conda info -e # find environment location +``` + +This might for example display the following path for the `vuegen_gui` environment: + +``` +/Users/user/miniforge3/envs/vuegen_gui +``` + +In the app, set the python environment path to this location, but to the `bin` folder, e.g. + +```bash +/Users/user/miniforge3/envs/vuegen_gui/bin +``` + +### virtualenv + +- tbc + +[virutalenv documentation](https://docs.python.org/3/library/venv.html) + +```bash +python -m venv .venv --copies --clear --prompt vuegenvenv +``` From b2b86aa62638fab36b208c849a39e80aa2f384a9 Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Wed, 12 Mar 2025 15:45:27 +0100 Subject: [PATCH 71/91] :bug: example data not in normal writable file path for bundled app, put config to output folder? - could be an option (to discuss) - config file is where output is, not where report - could be otherwise an option to be specified --- .github/workflows/cdci.yml | 4 ++-- src/vuegen/report_generator.py | 3 ++- src/vuegen/utils.py | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/cdci.yml b/.github/workflows/cdci.yml index 17b0cd9..066bd66 100644 --- a/.github/workflows/cdci.yml +++ b/.github/workflows/cdci.yml @@ -157,8 +157,8 @@ jobs: - name: Build executable run: | cd gui - pyinstaller -n vuegen_gui -D -w --collect-all pyvis --collect-all streamlit --collect-all st_aggrid --collect-all customtkinter --collect-all quarto_cli --collect-all plotly --collect-all _plotly_utils --collect-all traitlets --collect-all referencing --collect-all rpds --collect-all tenacity --collect-all pyvis --collect-all pandas --collect-all numpy --collect-all matplotlib --collect-all openpyxl --collect-all xlrd --collect-all nbformat --collect-all nbclient --collect-all altair --collect-all itables --collect-all kaleido --collect-all pyarrow --collect-all dataframe_image --collect-all narwhals --collect-all PIL --add-data ../docs/example_data/Basic_example_vuegen_demo_notebook:example_data/Basic_example_vuegen_demo_notebook app.py - # -w -F + pyinstaller -n vuegen_gui -F -w --collect-all pyvis --collect-all streamlit --collect-all st_aggrid --collect-all customtkinter --collect-all quarto_cli --collect-all plotly --collect-all _plotly_utils --collect-all traitlets --collect-all referencing --collect-all rpds --collect-all tenacity --collect-all pyvis --collect-all pandas --collect-all numpy --collect-all matplotlib --collect-all openpyxl --collect-all xlrd --collect-all nbformat --collect-all nbclient --collect-all altair --collect-all itables --collect-all kaleido --collect-all pyarrow --collect-all dataframe_image --collect-all narwhals --collect-all PIL --add-data ../docs/example_data/Basic_example_vuegen_demo_notebook:example_data/Basic_example_vuegen_demo_notebook app.py + # -w -D # --collect-all vl_convert --collect-all yaml --collect-all strenum --collect-all jinja2 --collect-all fastjsonschema --collect-all jsonschema --collect-all jsonschema_specifications # replace by spec file once done... - name: Upload executable diff --git a/src/vuegen/report_generator.py b/src/vuegen/report_generator.py index 7764cf1..860ef1b 100644 --- a/src/vuegen/report_generator.py +++ b/src/vuegen/report_generator.py @@ -56,7 +56,8 @@ def get_report( if dir_path: # Generate configuration from the provided directory yaml_data, base_folder_path = config_manager.create_yamlconfig_fromdir(dir_path) - config_path = write_yaml_config(yaml_data, base_folder_path) + config_path = write_yaml_config(yaml_data, output_dir) + logger.info("Configuration file generated at %s", config_path) # Load the YAML configuration file with the report metadata report_config = load_yaml_config(config_path) diff --git a/src/vuegen/utils.py b/src/vuegen/utils.py index 9404251..16d97aa 100644 --- a/src/vuegen/utils.py +++ b/src/vuegen/utils.py @@ -514,7 +514,7 @@ def write_yaml_config(yaml_data: dict, directory_path: Path) -> Path: raise FileNotFoundError(f"The directory {directory_path} does not exist.") # Now write the YAML file - with open(output_yaml, "w") as yaml_file: + with open(output_yaml, "w", encoding="utf-8") as yaml_file: yaml.dump(yaml_data, yaml_file, default_flow_style=False, sort_keys=False) # Return the path to the written file From 41363b1d3d07acc74a67abc9c83b2b4bb1dbaae9 Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Wed, 12 Mar 2025 17:58:40 +0100 Subject: [PATCH 72/91] :bug: svg from readme cannot be rendered --- .../5_Markdown/1_All_markdown/README.md | 48 ++++++++++--------- 1 file changed, 26 insertions(+), 22 deletions(-) diff --git a/docs/example_data/Basic_example_vuegen_demo_notebook/5_Markdown/1_All_markdown/README.md b/docs/example_data/Basic_example_vuegen_demo_notebook/5_Markdown/1_All_markdown/README.md index bb83be3..be93449 100644 --- a/docs/example_data/Basic_example_vuegen_demo_notebook/5_Markdown/1_All_markdown/README.md +++ b/docs/example_data/Basic_example_vuegen_demo_notebook/5_Markdown/1_All_markdown/README.md @@ -1,21 +1,23 @@ -![VueGen Logo](https://raw.githubusercontent.com/Multiomics-Analytics-Group/vuegen/main/docs/images/vuegen_logo.svg) ------------------ + +![VueGen Logo](https://raw.githubusercontent.com/Multiomics-Analytics-Group/vuegen/main/docs/images/vuegen_logo.png) +

VueGen is a Python library that automates the creation of scientific reports.

-| Information | Links | -| :--- | :--- | -| **Package** |[ ![PyPI Latest Release](https://img.shields.io/pypi/v/vuegen.svg)](https://pypi.org/project/vuegen/) [![Supported versions](https://img.shields.io/pypi/pyversions/vuegen.svg)](https://pypi.org/project/vuegen/)| -| **Documentation** | [![Docs](https://readthedocs.org/projects/vuegen/badge/?style=flat)](https://vuegen.readthedocs.io/)| -| **Build** | [![CI](https://github.com/Multiomics-Analytics-Group/vuegen/actions/workflows/cdci.yml/badge.svg)](https://github.com/Multiomics-Analytics-Group/vuegen/actions/workflows/cdci.yml) [![Docs](https://github.com/Multiomics-Analytics-Group/vuegen/actions/workflows/docs.yml/badge.svg)](https://github.com/Multiomics-Analytics-Group/vuegen/actions/workflows/docs.yml)| -| **Examples** | [![HTML5](https://img.shields.io/badge/html5-%23E34F26.svg?style=for-the-badge&logo=html5&logoColor=white)](https://multiomics-analytics-group.github.io/vuegen/) [![Streamlit](https://img.shields.io/badge/Streamlit-%23FE4B4B.svg?style=for-the-badge&logo=streamlit&logoColor=white)](https://multiomics-analytics-group.github.io/vuegen/)| -| **Discuss on GitHub** | [![GitHub issues](https://img.shields.io/github/issues/Multiomics-Analytics-Group/vuegen)](https://github.com/Multiomics-Analytics-Group/vuegen/issues) [![GitHub pull requests](https://img.shields.io/github/issues-pr/Multiomics-Analytics-Group/vuegen)](https://github.com/Multiomics-Analytics-Group/vuegen/pulls) | +| Information | Links | +| :-------------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| **Package** | [PyPI Latest Release](https://pypi.org/project/vuegen/) [Supported versions](https://pypi.org/project/vuegen/) | +| **Documentation** | [Docs](https://vuegen.readthedocs.io/) | +| **Build** | [CI](https://github.com/Multiomics-Analytics-Group/vuegen/actions/workflows/cdci.yml) - [Docs](https://github.com/Multiomics-Analytics-Group/vuegen/actions/workflows/docs.yml) | +| **Examples** | [HTML5](https://multiomics-analytics-group.github.io/vuegen/) - [Streamlit](https://multiomics-analytics-group.github.io/vuegen/) | +| **Discuss on GitHub** | [GitHub issues](https://github.com/Multiomics-Analytics-Group/vuegen/issues) - [GitHub pull requests](https://github.com/Multiomics-Analytics-Group/vuegen/pulls) | ## Table of contents: + - [About the project](#about-the-project) - [Installation](#installation) - [Execution](#execution) @@ -23,7 +25,8 @@ - [Contact](#contact) ## About the project -VueGen automates the creation of reports based on a directory with plots, dataframes, and other files in different formats. A YAML configuration file is generated from the directory to define the structure of the report. Users can customize the report by modifying the configuration file, or they can create their own configuration file instead of passing a directory as input. + +VueGen automates the creation of reports based on a directory with plots, dataframes, and other files in different formats. A YAML configuration file is generated from the directory to define the structure of the report. Users can customize the report by modifying the configuration file, or they can create their own configuration file instead of passing a directory as input. The configuration file specifies the structure of the report, including sections, subsections, and various components such as plots, dataframes, markdown, html, and API calls. Reports can be generated in various formats, including documents (PDF, HTML, DOCX, ODT), presentations (PPTX, Reveal.js), notebooks (Jupyter) or [Streamlit](streamlit) web applications. @@ -34,6 +37,7 @@ An overview of the VueGen workflow is shown in the figure below: VueGen overview

--> + ![VueGen Abstract](https://raw.githubusercontent.com/Multiomics-Analytics-Group/vuegen/main/docs/images/vuegen_graph_abstract.png) Also, the class diagram for the project is presented below to illustrate the architecture and relationships between classes: @@ -57,7 +61,7 @@ pip install vuegen You can also install the package for development from this repository by running the following command: ```bash -pip install -e path/to/vuegen # specify location +pip install -e path/to/vuegen # specify location pip install -e . # in case your pwd is in the vuegen directory ``` @@ -91,14 +95,15 @@ vuegen --config example_data/Earth_microbiome_vuegen_demo_notebook/Earth_microbi ``` The current report types supported by VueGen are: -* Streamlit -* HTML -* PDF -* DOCX -* ODT -* Reveal.js -* PPTX -* Jupyter + +- Streamlit +- HTML +- PDF +- DOCX +- ODT +- Reveal.js +- PPTX +- Jupyter ## Acknowledgements @@ -106,9 +111,10 @@ The current report types supported by VueGen are: - The vuegen logo was designed based on an image created by [Scriberia][scriberia] for The [Turing Way Community][turingway], which is shared under a CC-BY licence. The original image can be found at [Zenodo][zenodo-turingway]. ## Contact + If you have comments or suggestions about this project, you can [open an issue][issues] in this repository. -[streamlit]: https://streamlit.io/ +[streamlit]: https://streamlit.io/ [vuegen-pypi]: https://pypi.org/project/vuegen/ [quarto]: https://quarto.org/ [quarto-cli-pypi]: https://pypi.org/project/quarto-cli/ @@ -119,5 +125,3 @@ If you have comments or suggestions about this project, you can [open an issue][ [turingway]: https://github.com/the-turing-way/the-turing-way [zenodo-turingway]: https://zenodo.org/records/3695300 [issues]: https://github.com/Multiomics-Analytics-Group/vuegen/issues/new - - From c2d1fbf3843da273c251d8fe2549e6d3e30718a4 Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Wed, 12 Mar 2025 18:00:16 +0100 Subject: [PATCH 73/91] :bug: reset logger (relevant for GUI) - logging.basicConfig only has an effect the first time - executed from GUI, the logs were created but empty the second time --- src/vuegen/utils.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/src/vuegen/utils.py b/src/vuegen/utils.py index 16d97aa..e86e92a 100644 --- a/src/vuegen/utils.py +++ b/src/vuegen/utils.py @@ -690,17 +690,22 @@ def init_log( else: handlers = [file_handler] - # logger configuration - logging.basicConfig( - # level=logging.DEBUG, - format="[%(asctime)s] %(name)s: %(levelname)s - %(message)s", - handlers=handlers, - ) - logging.getLogger("matplotlib.font_manager").disabled = True - # instantiate the logger logger = logging.getLogger(logger_id) logger.setLevel(logging.DEBUG) + # logger configuration + # ! logging.basicConfig has no effect if called once anywhere in the code + # ! set handlers and format for the logger manually + # Reset any existing handlers + for handler in logging.root.handlers[:]: + logger.removeHandler(handler) + + # Set up the new handlers and format + formatter = logging.Formatter("[%(asctime)s] %(name)s: %(levelname)s - %(message)s") + for handler in handlers: + handler.setFormatter(formatter) + logger.addHandler(handler) + logging.getLogger("matplotlib.font_manager").disabled = True return logger From f300e1abb8bedab368207cdb7f5214e3797474ac Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Thu, 13 Mar 2025 15:13:50 +0100 Subject: [PATCH 74/91] :boom: Write config file based on directory to output directory of report MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - read-only situation of result directories: One cannot always write to directory which contains the results if -dir option is used - ✨ Return paths for easier reporting - adapt example notebooks accordingly --- docs/vuegen_basic_case_study.ipynb | 33 ++++++------ docs/vuegen_case_study_earth_microbiome.ipynb | 53 ++++++++++--------- gui/app.py | 12 +++-- src/vuegen/__main__.py | 3 +- src/vuegen/report_generator.py | 26 ++++++--- src/vuegen/utils.py | 3 +- 6 files changed, 73 insertions(+), 57 deletions(-) diff --git a/docs/vuegen_basic_case_study.ipynb b/docs/vuegen_basic_case_study.ipynb index 1268256..ff3218b 100644 --- a/docs/vuegen_basic_case_study.ipynb +++ b/docs/vuegen_basic_case_study.ipynb @@ -73,7 +73,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -164,22 +164,18 @@ "source": [ "# Generate the report\n", "report_type = \"streamlit\"\n", - "report_generator.get_report(dir_path = base_output_dir, report_type = report_type, logger = None)" + "report_dir, config_path = report_generator.get_report(\n", + " dir_path=base_output_dir, report_type=report_type, logger=None\n", + ")\n", + "print(f\"\\nReport generated in {report_dir}\")\n", + "print(f\"\\nConfig file generated in {config_path}\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Streamlit report not executed, set run_streamlit to True to run the report\n" - ] - } - ], + "outputs": [], "source": [ "run_streamlit = False\n", "# run_streamlit = True # uncomment line to run the streamlit report\n", @@ -212,7 +208,8 @@ "source": [ "# Generate the report\n", "report_type = \"html\"\n", - "report_generator.get_report(dir_path = base_output_dir, report_type = report_type, logger = None)" + "report_dir, config_path = report_generator.get_report(dir_path = base_output_dir, report_type = report_type, logger = None)\n", + "print(f\"Report generated at: {report_dir}\")" ] }, { @@ -232,14 +229,14 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ "vuegen_logo_path = \"https://raw.githubusercontent.com/Multiomics-Analytics-Group/vuegen/main/docs/images/vuegen_logo.svg\"\n", "\n", "# Load the YAML file\n", - "config_path = os.path.join(base_output_dir, \"Basic_example_vuegen_demo_notebook_config.yaml\")\n", + "print(f\"Loading the YAML config file from: {config_path}\") # generated based on directory path above\n", "config = load_yaml_config(config_path)\n", "\n", "# Update the logo and graphical abstract with the URL\n", @@ -255,7 +252,7 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -281,7 +278,7 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -324,7 +321,7 @@ "source": [ "# Test the changes by generarating the report from the modified YAML file\n", "report_type = \"streamlit\"\n", - "report_generator.get_report(config_path = config_path, report_type = report_type, logger = None)" + "_ = report_generator.get_report(config_path = config_path, report_type = report_type, logger = None)" ] }, { @@ -364,7 +361,7 @@ "source": [ "# Test the changes by generarating the report from the modified YAML file\n", "report_type = \"html\"\n", - "report_generator.get_report(config_path = config_path, report_type = report_type, logger = None)" + "_ = report_generator.get_report(config_path = config_path, report_type = report_type, logger = None)" ] } ], diff --git a/docs/vuegen_case_study_earth_microbiome.ipynb b/docs/vuegen_case_study_earth_microbiome.ipynb index 1dff5eb..b9ddad4 100644 --- a/docs/vuegen_case_study_earth_microbiome.ipynb +++ b/docs/vuegen_case_study_earth_microbiome.ipynb @@ -124,7 +124,7 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -179,7 +179,7 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -207,7 +207,7 @@ }, { "cell_type": "code", - "execution_count": 30, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -281,7 +281,7 @@ }, { "cell_type": "code", - "execution_count": 31, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -304,7 +304,7 @@ }, { "cell_type": "code", - "execution_count": 32, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -536,7 +536,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -551,7 +551,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -759,7 +759,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -776,7 +776,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -914,7 +914,7 @@ }, { "cell_type": "code", - "execution_count": 33, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -943,7 +943,7 @@ }, { "cell_type": "code", - "execution_count": 34, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -1193,7 +1193,7 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -1215,7 +1215,7 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -1237,7 +1237,7 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -1257,7 +1257,7 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -1277,7 +1277,7 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -1294,7 +1294,7 @@ }, { "cell_type": "code", - "execution_count": 30, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -1382,7 +1382,7 @@ "source": [ "# Generate the report\n", "report_type = \"streamlit\"\n", - "report_generator.get_report(dir_path = base_output_dir, report_type = report_type, logger = None)" + "_ = report_generator.get_report(dir_path = base_output_dir, report_type = report_type, logger = None)" ] }, { @@ -1422,7 +1422,9 @@ "source": [ "# Generate the report\n", "report_type = \"html\"\n", - "report_generator.get_report(dir_path = base_output_dir, report_type = report_type, logger = None)" + "report_dir, config_path = report_generator.get_report(\n", + " dir_path=base_output_dir, report_type=report_type, logger=None\n", + ")" ] }, { @@ -1442,14 +1444,13 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ "empo_logo_path = \"https://raw.githubusercontent.com/ElDeveloper/cogs220/master/emp-logo.svg\"\n", "\n", "# Load the YAML file\n", - "config_path = os.path.join(base_output_dir, \"Earth_microbiome_vuegen_demo_notebook_config.yaml\")\n", "config = load_yaml_config(config_path)\n", "\n", "# Update the logo and graphical abstract with the URL\n", @@ -1465,7 +1466,7 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -1491,7 +1492,7 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -1522,7 +1523,7 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -1570,7 +1571,7 @@ "source": [ "# Test the changes by generarating the report from the modified YAML file\n", "report_type = \"streamlit\"\n", - "report_generator.get_report(config_path = config_path, report_type = report_type, logger = None)" + "_ = report_generator.get_report(config_path = config_path, report_type = report_type, logger = None)" ] }, { @@ -1610,7 +1611,7 @@ "source": [ "# Test the changes by generarating the report from the modified YAML file\n", "report_type = \"html\"\n", - "report_generator.get_report(config_path = config_path, report_type = report_type, logger = None)" + "_ = report_generator.get_report(config_path = config_path, report_type = report_type, logger = None)" ] } ], diff --git a/gui/app.py b/gui/app.py index f2b0f01..94294ef 100644 --- a/gui/app.py +++ b/gui/app.py @@ -53,6 +53,8 @@ app_path.parent / "example_data/Basic_example_vuegen_demo_notebook" ).resolve() quarto_bin_path = os.path.join(sys._MEIPASS, "quarto_cli", "bin") + # /.venv/lib/python3.12/site-packages/quarto_cli/bin + # source activate .venv/bin/activate quarto_share_path = os.path.join(sys._MEIPASS, "quarto_cli", "share") _PATH = os.pathsep.join([quarto_bin_path, quarto_share_path, _PATH]) os.environ["PATH"] = _PATH @@ -146,10 +148,14 @@ def inner(): kwargs["logger"].info("logfile: %s", log_file) kwargs["logger"].debug("sys.path: %s", sys.path) kwargs["logger"].debug("PATH (in app): %s ", os.environ["PATH"]) - report_generator.get_report(**kwargs) + report_dir, gen_config_path = report_generator.get_report(**kwargs) + kwargs["logger"].info("Report generated at %s", report_dir) messagebox.showinfo( "Success", - "Report generation completed successfully." f"\nLogs at {log_file}", + "Report generation completed successfully." + f"\n\nLogs at:\n{log_file}" + f"\n\nReport in folder:\n{report_dir}" + f"\n\nConfiguration file at:\n{gen_config_path}", ) print_completion_message(report_type.get()) except Exception as e: @@ -245,7 +251,7 @@ def select_directory(): ctk_label_report.grid(row=row_count, column=0, columnspan=2, padx=20, pady=20) row_count += 1 ########################################################################################## -report_type = tk.StringVar(value=report_types[0]) +report_type = tk.StringVar(value=report_types[1]) report_dropdown = customtkinter.CTkOptionMenu( app, values=report_types, diff --git a/src/vuegen/__main__.py b/src/vuegen/__main__.py index e911a59..100542f 100644 --- a/src/vuegen/__main__.py +++ b/src/vuegen/__main__.py @@ -39,7 +39,7 @@ def main(): logger, logfile = get_logger(f"{logger_suffix}") logger.info("logfile: %s", logfile) # Generate the report - report_generator.get_report( + _, _ = report_generator.get_report( report_type=report_type, logger=logger, config_path=config_path, @@ -48,6 +48,7 @@ def main(): ) # Print completion message + # ! Could use now report_dir and config_path as information print_completion_message(report_type) diff --git a/src/vuegen/report_generator.py b/src/vuegen/report_generator.py index 860ef1b..c77d0c8 100644 --- a/src/vuegen/report_generator.py +++ b/src/vuegen/report_generator.py @@ -17,7 +17,7 @@ def get_report( dir_path: str = None, streamlit_autorun: bool = False, output_dir: Path = None, -) -> None: +) -> tuple[str, str]: """ Generate and run a report based on the specified engine. @@ -38,6 +38,11 @@ def get_report( ------ ValueError If neither 'config_path' nor 'directory' is provided. + + Returns + ------- + tuple[str, str] + The path to the generated report and the path to the configuration file. """ if output_dir is None: output_dir = Path(".") @@ -56,6 +61,7 @@ def get_report( if dir_path: # Generate configuration from the provided directory yaml_data, base_folder_path = config_manager.create_yamlconfig_fromdir(dir_path) + # yaml_data has under report a title created based on the directory name config_path = write_yaml_config(yaml_data, output_dir) logger.info("Configuration file generated at %s", config_path) @@ -70,9 +76,9 @@ def get_report( # Create and run ReportView object based on its type if report_type == ReportType.STREAMLIT: - base_dir = output_dir / "streamlit_report" - sections_dir = base_dir / "sections" - static_files_dir = base_dir / "static" + report_dir = output_dir / "streamlit_report" + sections_dir = report_dir / "sections" + static_files_dir = report_dir / "static" st_report = StreamlitReportView( report=report, report_type=report_type, streamlit_autorun=streamlit_autorun ) @@ -89,8 +95,12 @@ def get_report( raise RuntimeError( "Quarto is not installed. Please install Quarto before generating this report type." ) - base_dir = output_dir / "quarto_report" - static_files_dir = base_dir / "static" + report_dir = output_dir / "quarto_report" + static_files_dir = report_dir / "static" quarto_report = QuartoReportView(report=report, report_type=report_type) - quarto_report.generate_report(output_dir=base_dir, static_dir=static_files_dir) - quarto_report.run_report(output_dir=base_dir) + quarto_report.generate_report( + output_dir=report_dir, static_dir=static_files_dir + ) + quarto_report.run_report(output_dir=report_dir) + # ? Could be also the path to the report file for quarto based reports + return report_dir, config_path diff --git a/src/vuegen/utils.py b/src/vuegen/utils.py index e86e92a..4f3b43a 100644 --- a/src/vuegen/utils.py +++ b/src/vuegen/utils.py @@ -507,7 +507,8 @@ def write_yaml_config(yaml_data: dict, directory_path: Path) -> Path: assert isinstance(directory_path, Path), "directory_path must be a Path object." # Generate the output YAML file path based on the folder name - output_yaml = directory_path / (directory_path.name + "_config.yaml") + _name = yaml_data["report"]["title"].replace(" ", "_").lower() + output_yaml = directory_path / f"{_name}_config.yaml" # Ensure the directory exists (but don't create a new folder) if not directory_path.exists(): From f3eef5c696ff4b9cf4de8c04c36fba0073082292 Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Thu, 13 Mar 2025 15:51:04 +0100 Subject: [PATCH 75/91] :sparkles: improve folder dialog for PythonPath, set str explicitly for Windows - On windows the Path set by the user globally are probably seen (it seems) - check if a path is set for directory dialog, if not start at home directory --- gui/app.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/gui/app.py b/gui/app.py index 94294ef..6d05975 100644 --- a/gui/app.py +++ b/gui/app.py @@ -127,7 +127,9 @@ def inner(): [ quarto_bin_path, quarto_share_path, - python_dir_entry.get(), + str( + Path(python_dir_entry.get()) + ), # ! check if this return WindowsPaths on Windows _PATH, ] ) @@ -183,7 +185,10 @@ def radio_button_callback(): def create_select_directory(string_var): def select_directory(): - directory = filedialog.askdirectory(initialdir=string_var.get()) + inital_dir = string_var.get() + if not inital_dir: + inital_dir = Path.home() + directory = filedialog.askdirectory(initialdir=inital_dir) string_var.set(directory) return select_directory From dbe96eda4884330db7c51e70057906d5d049f760 Mon Sep 17 00:00:00 2001 From: enryh Date: Fri, 14 Mar 2025 11:34:32 +0100 Subject: [PATCH 76/91] :art: set default logger_id (so not all loggers are set to debug) --- src/vuegen/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/vuegen/utils.py b/src/vuegen/utils.py index 4f3b43a..875a234 100644 --- a/src/vuegen/utils.py +++ b/src/vuegen/utils.py @@ -711,7 +711,7 @@ def init_log( return logger -def get_logger(log_suffix, folder="logs", display=True) -> tuple[logging.Logger, str]: +def get_logger(log_suffix, folder="logs", display=True, logger_id='vuegen') -> tuple[logging.Logger, str]: """ Initialize the logger with a log file name that includes an optional suffix. @@ -729,7 +729,7 @@ def get_logger(log_suffix, folder="logs", display=True) -> tuple[logging.Logger, log_file = generate_log_filename(folder=folder, suffix=log_suffix) # Initialize logger - logger = init_log(log_file, display=display) + logger = init_log(log_file, display=display, logger_id=logger_id) # Log the path to the log file logger.info(f"Path to log file: {log_file}") From 8222d2b07f93c20f53edc6446b1d08c94f7bce23 Mon Sep 17 00:00:00 2001 From: enryh Date: Fri, 14 Mar 2025 11:35:30 +0100 Subject: [PATCH 77/91] :bug: do not overwrite _PATH - otherwise bundled quarto is set twice if python env is selected --- gui/app.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/gui/app.py b/gui/app.py index 6d05975..e6a3831 100644 --- a/gui/app.py +++ b/gui/app.py @@ -56,8 +56,7 @@ # /.venv/lib/python3.12/site-packages/quarto_cli/bin # source activate .venv/bin/activate quarto_share_path = os.path.join(sys._MEIPASS, "quarto_cli", "share") - _PATH = os.pathsep.join([quarto_bin_path, quarto_share_path, _PATH]) - os.environ["PATH"] = _PATH + os.environ["PATH"] = os.pathsep.join([quarto_bin_path, quarto_share_path, _PATH]) # This requires that the python version is the same as the one used to create the executable # in the Python environment the kernel is started from for quarto based reports # os.environ["PYTHONPATH"] = os.pathsep.join( From 65465b90d7ae1d156d0e0f418547fb7676b1d2e7 Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Fri, 14 Mar 2025 13:23:32 +0100 Subject: [PATCH 78/91] :art: add logo to app and clean-up app.py - for debugging do not build windowed app for now --- .github/workflows/cdci.yml | 7 ++++-- gui/Makefile | 1 + gui/app.py | 49 ++++++++++++++++---------------------- 3 files changed, 26 insertions(+), 31 deletions(-) diff --git a/.github/workflows/cdci.yml b/.github/workflows/cdci.yml index 066bd66..0dc4aae 100644 --- a/.github/workflows/cdci.yml +++ b/.github/workflows/cdci.yml @@ -157,8 +157,11 @@ jobs: - name: Build executable run: | cd gui - pyinstaller -n vuegen_gui -F -w --collect-all pyvis --collect-all streamlit --collect-all st_aggrid --collect-all customtkinter --collect-all quarto_cli --collect-all plotly --collect-all _plotly_utils --collect-all traitlets --collect-all referencing --collect-all rpds --collect-all tenacity --collect-all pyvis --collect-all pandas --collect-all numpy --collect-all matplotlib --collect-all openpyxl --collect-all xlrd --collect-all nbformat --collect-all nbclient --collect-all altair --collect-all itables --collect-all kaleido --collect-all pyarrow --collect-all dataframe_image --collect-all narwhals --collect-all PIL --add-data ../docs/example_data/Basic_example_vuegen_demo_notebook:example_data/Basic_example_vuegen_demo_notebook app.py - # -w -D + pyinstaller -n vuegen_gui --collect-all pyvis --collect-all streamlit --collect-all st_aggrid --collect-all customtkinter --collect-all quarto_cli --collect-all plotly --collect-all _plotly_utils --collect-all traitlets --collect-all referencing --collect-all rpds --collect-all tenacity --collect-all pyvis --collect-all pandas --collect-all numpy --collect-all matplotlib --collect-all openpyxl --collect-all xlrd --collect-all nbformat --collect-all nbclient --collect-all altair --collect-all itables --collect-all kaleido --collect-all pyarrow --collect-all dataframe_image --collect-all narwhals --collect-all PIL --add-data ../docs/example_data/Basic_example_vuegen_demo_notebook:example_data/Basic_example_vuegen_demo_notebook --add-data ../docs/images/vuegen_logo.png:. app.py + # --windowed only for mac, see: + # https://pyinstaller.org/en/stable/usage.html#building-macos-app-bundles + # 'Under macOS, PyInstaller always builds a UNIX executable in dist.' + # --onefile --windowed for Windows? # --collect-all vl_convert --collect-all yaml --collect-all strenum --collect-all jinja2 --collect-all fastjsonschema --collect-all jsonschema --collect-all jsonschema_specifications # replace by spec file once done... - name: Upload executable diff --git a/gui/Makefile b/gui/Makefile index 73b01ba..384212a 100644 --- a/gui/Makefile +++ b/gui/Makefile @@ -37,6 +37,7 @@ bundle: --collect-all rpds \ --collect-all tenacity \ --add-data ../docs/example_data/Basic_example_vuegen_demo_notebook:example_data/Basic_example_vuegen_demo_notebook \ + --add-data ../docs/images/vuegen_logo.png:. \ app.py diff --git a/gui/app.py b/gui/app.py index e6a3831..e1f145b 100644 --- a/gui/app.py +++ b/gui/app.py @@ -22,22 +22,22 @@ import os import sys import tkinter as tk +import traceback from pathlib import Path from pprint import pprint +from tkinter import filedialog, messagebox import customtkinter +from PIL import Image + +from vuegen import report_generator # from vuegen.__main__ import main from vuegen.report import ReportType -from vuegen.utils import get_logger +from vuegen.utils import get_logger, print_completion_message customtkinter.set_appearance_mode("system") customtkinter.set_default_color_theme("dark-blue") -import traceback -from tkinter import filedialog, messagebox - -from vuegen import report_generator -from vuegen.utils import print_completion_message app_path = Path(__file__).absolute().resolve() print("app_path:", app_path) @@ -64,41 +64,24 @@ # ) # ! requires kernel env with same Python env, but does not really seem to help os.environ["PYTHONPATH"] = sys._MEIPASS # ([[sys.path[0], sys._MEIPASS]) # does not work when built on GitHub Actions + logo_path = os.path.join(sys._MEIPASS, "vuegen_logo.png") elif app_path.parent.name == "gui": # should be always the case for GUI run from command line path_to_dat = ( - app_path.parent - / ".." + app_path.parent.parent / "docs" / "example_data" / "Basic_example_vuegen_demo_notebook" ).resolve() + logo_path = ( + app_path.parent.parent / "docs" / "images" / "vuegen_logo.png" + ) # 1000x852 pixels else: path_to_dat = "docs/example_data/Basic_example_vuegen_demo_notebook" print(f"{_PATH = }") ########################################################################################## # callbacks -# using main entry point of vuegen -# def create_run_vuegen(is_dir, config_path, report_type, run_streamlit): -# def inner(): -# args = ["vuegen"] -# print(f"{is_dir.get() = }") -# if is_dir.get(): -# args.append("--directory") -# else: -# args.append("--config") -# args.append(config_path.get()) -# args.append("--report_type") -# args.append(report_type.get()) -# print(f"{run_streamlit.get() = }") -# if run_streamlit.get(): -# args.append("--streamlit_autorun") -# print("args:", args) -# sys.argv = args -# main() # Call the main function from vuegen - -# return inner def create_run_vuegen( @@ -198,9 +181,17 @@ def select_directory(): app = customtkinter.CTk() app.geometry("620x600") app.title("VueGen GUI") - row_count = 0 ########################################################################################## +# Logo +_factor = 4 +logo_image = customtkinter.CTkImage( + Image.open(logo_path), size=(int(1000 / _factor), int(852 / _factor)) +) +logo_label = customtkinter.CTkLabel(app, image=logo_image, text="") +logo_label.grid(row=0, column=0, columnspan=2, padx=10, pady=5) +row_count += 1 +########################################################################################## # Config or directory input ctk_label_config = customtkinter.CTkLabel( app, From 0c8597454bc660c591f8d88be56f3241e0c6649a Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Mon, 17 Mar 2025 09:02:19 +0100 Subject: [PATCH 79/91] :art: do not reset directories when nothing is set, adj. app size --- gui/app.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/gui/app.py b/gui/app.py index e1f145b..9d5ce1d 100644 --- a/gui/app.py +++ b/gui/app.py @@ -171,7 +171,8 @@ def select_directory(): if not inital_dir: inital_dir = Path.home() directory = filedialog.askdirectory(initialdir=inital_dir) - string_var.set(directory) + if directory: + string_var.set(directory) return select_directory @@ -179,7 +180,7 @@ def select_directory(): ########################################################################################## # APP app = customtkinter.CTk() -app.geometry("620x600") +app.geometry("620x840") app.title("VueGen GUI") row_count = 0 ########################################################################################## From 2f86c1e91ef17b43beb5a1416f728ddce16c8971 Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Mon, 17 Mar 2025 09:03:42 +0100 Subject: [PATCH 80/91] :sparkles: cache python exectutable set previously - only require users to setup python environment once - :memo: start describing how to use bundled app --- gui/README.md | 15 +++++++++++++-- gui/app.py | 19 ++++++++++++++++++- 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/gui/README.md b/gui/README.md index 3456a69..bd486b7 100644 --- a/gui/README.md +++ b/gui/README.md @@ -76,7 +76,18 @@ Windows and macOS specific options: - maybe a self-contained minimal virtual environment for kernel starting can be added later - we could add some logic to make sure a correct path is added. -### Create environment using conda +## Using bundle vuegen release + +## On Windows + +- global quarto and python installations can be used +- quarto can be shipped with app, but maybe it can be deactivated + +## On MacOs + +- on MacOs the default paths are not set + +#### Create environment using conda ```bash conda create -n vuegen_gui -c conda-forge python=3.12 jupyter @@ -95,7 +106,7 @@ In the app, set the python environment path to this location, but to the `bin` f /Users/user/miniforge3/envs/vuegen_gui/bin ``` -### virtualenv +#### virtualenv - tbc diff --git a/gui/app.py b/gui/app.py index 9d5ce1d..56b8328 100644 --- a/gui/app.py +++ b/gui/app.py @@ -28,6 +28,7 @@ from tkinter import filedialog, messagebox import customtkinter +import yaml from PIL import Image from vuegen import report_generator @@ -45,6 +46,15 @@ print("output_dir:", output_dir) output_dir.mkdir(exist_ok=True, parents=True) _PATH = f'{os.environ["PATH"]}' +### config path for app +config_file = Path(Path.home() / ".vuegen_gui" / "config.yaml").resolve() +if not config_file.exists(): + config_file.parent.mkdir(exist_ok=True, parents=True) + config_app = dict(python_dir_entry="") +else: + with open(config_file, "r", encoding="utf-8") as f: + config_app = yaml.safe_load(f) +hash_config_app = hash(yaml.dump(config_app)) ########################################################################################## # Path to example data dependend on how the GUI is run if getattr(sys, "frozen", False) and hasattr(sys, "_MEIPASS"): @@ -104,6 +114,8 @@ def inner(): pprint(kwargs) if python_dir_entry.get(): + if python_dir_entry.get() != config_app["python_dir_entry"]: + config_app["python_dir_entry"] = python_dir_entry.get() if getattr(sys, "frozen", False) and hasattr(sys, "_MEIPASS"): os.environ["PATH"] = os.pathsep.join( [ @@ -141,7 +153,12 @@ def inner(): f"\n\nReport in folder:\n{report_dir}" f"\n\nConfiguration file at:\n{gen_config_path}", ) + global hash_config_app # ! fix this print_completion_message(report_type.get()) + if hash(yaml.dump(config_app)) != hash_config_app: + with open(config_file, "w", encoding="utf-8") as f: + yaml.dump(config_app, f) + hash_config_app = hash(yaml.dump(config_app)) except Exception as e: stacktrace = traceback.format_exc() messagebox.showerror( @@ -305,7 +322,7 @@ def select_directory(): ctk_label_outdir.grid(row=row_count, column=0, columnspan=1, padx=10, pady=5) row_count += 1 ########################################################################################## -python_dir_entry = tk.StringVar(value="") +python_dir_entry = tk.StringVar(value=config_app["python_dir_entry"]) select_python_bin = create_select_directory(python_dir_entry) select_python_bin_button = customtkinter.CTkButton( app, text="Select Python binary", command=select_python_bin From fafc95c4ee8da62ab2db8f416061ce0c5dff817c Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Mon, 17 Mar 2025 12:47:39 +0100 Subject: [PATCH 81/91] :art: format, escape pagemark --- src/vuegen/quarto_reportview.py | 2 +- src/vuegen/utils.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/vuegen/quarto_reportview.py b/src/vuegen/quarto_reportview.py index 94a606d..308a045 100644 --- a/src/vuegen/quarto_reportview.py +++ b/src/vuegen/quarto_reportview.py @@ -282,7 +282,7 @@ def _create_yaml_header(self) -> str: \\usepackage{hyperref} \\clearpairofpagestyles \\lofoot{This report was generated with \\href{https://github.com/Multiomics-Analytics-Group/vuegen}{VueGen} | \\copyright{} 2025 \\href{https://github.com/Multiomics-Analytics-Group}{Multiomics Network Analytics Group}} - \\rofoot{\pagemark}""", + \\rofoot{\\pagemark}""", r.ReportType.DOCX: """ docx: toc: false""", diff --git a/src/vuegen/utils.py b/src/vuegen/utils.py index 875a234..61358a2 100644 --- a/src/vuegen/utils.py +++ b/src/vuegen/utils.py @@ -711,7 +711,9 @@ def init_log( return logger -def get_logger(log_suffix, folder="logs", display=True, logger_id='vuegen') -> tuple[logging.Logger, str]: +def get_logger( + log_suffix, folder="logs", display=True, logger_id="vuegen" +) -> tuple[logging.Logger, str]: """ Initialize the logger with a log file name that includes an optional suffix. From 576a36e1b8375b7dfafc2a9cc5197e3dfe6b6888 Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Mon, 17 Mar 2025 15:40:00 +0100 Subject: [PATCH 82/91] :sparkles: copy example data to vuegen_gui dir in home directory, lib for png export of altair fig - address windows pandoc issue: "pandoc: openBinaryFile: does not exist (No such file or directory)" - add library vl-convert-python needed for altair fig exported as static png --- .github/workflows/cdci.yml | 4 ++-- gui/Makefile | 2 +- gui/app.py | 23 +++++++++++++++++++---- 3 files changed, 22 insertions(+), 7 deletions(-) diff --git a/.github/workflows/cdci.yml b/.github/workflows/cdci.yml index 0dc4aae..dc6704c 100644 --- a/.github/workflows/cdci.yml +++ b/.github/workflows/cdci.yml @@ -157,12 +157,12 @@ jobs: - name: Build executable run: | cd gui - pyinstaller -n vuegen_gui --collect-all pyvis --collect-all streamlit --collect-all st_aggrid --collect-all customtkinter --collect-all quarto_cli --collect-all plotly --collect-all _plotly_utils --collect-all traitlets --collect-all referencing --collect-all rpds --collect-all tenacity --collect-all pyvis --collect-all pandas --collect-all numpy --collect-all matplotlib --collect-all openpyxl --collect-all xlrd --collect-all nbformat --collect-all nbclient --collect-all altair --collect-all itables --collect-all kaleido --collect-all pyarrow --collect-all dataframe_image --collect-all narwhals --collect-all PIL --add-data ../docs/example_data/Basic_example_vuegen_demo_notebook:example_data/Basic_example_vuegen_demo_notebook --add-data ../docs/images/vuegen_logo.png:. app.py + pyinstaller -n vuegen_gui --collect-all pyvis --collect-all streamlit --collect-all st_aggrid --collect-all customtkinter --collect-all quarto_cli --collect-all plotly --collect-all _plotly_utils --collect-all traitlets --collect-all referencing --collect-all rpds --collect-all tenacity --collect-all pyvis --collect-all pandas --collect-all numpy --collect-all matplotlib --collect-all openpyxl --collect-all xlrd --collect-all nbformat --collect-all nbclient --collect-all altair --collect-all itables --collect-all kaleido --collect-all pyarrow --collect-all dataframe_image --collect-all narwhals --collect-all PIL --collect-all vl_convert --add-data ../docs/example_data/Basic_example_vuegen_demo_notebook:example_data/Basic_example_vuegen_demo_notebook --add-data ../docs/images/vuegen_logo.png:. app.py # --windowed only for mac, see: # https://pyinstaller.org/en/stable/usage.html#building-macos-app-bundles # 'Under macOS, PyInstaller always builds a UNIX executable in dist.' # --onefile --windowed for Windows? - # --collect-all vl_convert --collect-all yaml --collect-all strenum --collect-all jinja2 --collect-all fastjsonschema --collect-all jsonschema --collect-all jsonschema_specifications + # --collect-all yaml --collect-all strenum --collect-all jinja2 --collect-all fastjsonschema --collect-all jsonschema --collect-all jsonschema_specifications # replace by spec file once done... - name: Upload executable uses: actions/upload-artifact@v4 diff --git a/gui/Makefile b/gui/Makefile index 384212a..57faced 100644 --- a/gui/Makefile +++ b/gui/Makefile @@ -36,13 +36,13 @@ bundle: --collect-all referencing \ --collect-all rpds \ --collect-all tenacity \ + --collect-all vl_convert \ --add-data ../docs/example_data/Basic_example_vuegen_demo_notebook:example_data/Basic_example_vuegen_demo_notebook \ --add-data ../docs/images/vuegen_logo.png:. \ app.py # jupyter kernel specific: -# --collect-all vl_convert \ # --collect-all yaml \ # --collect-all strenum \ # --collect-all jinja2 \ diff --git a/gui/app.py b/gui/app.py index 56b8328..a9a2d9d 100644 --- a/gui/app.py +++ b/gui/app.py @@ -20,6 +20,7 @@ """ import os +import shutil import sys import tkinter as tk import traceback @@ -59,7 +60,7 @@ # Path to example data dependend on how the GUI is run if getattr(sys, "frozen", False) and hasattr(sys, "_MEIPASS"): # PyInstaller bundeled case - path_to_dat = ( + path_to_data_in_bundle = ( app_path.parent / "example_data/Basic_example_vuegen_demo_notebook" ).resolve() quarto_bin_path = os.path.join(sys._MEIPASS, "quarto_cli", "bin") @@ -74,10 +75,24 @@ # ) # ! requires kernel env with same Python env, but does not really seem to help os.environ["PYTHONPATH"] = sys._MEIPASS # ([[sys.path[0], sys._MEIPASS]) # does not work when built on GitHub Actions + path_to_example_data = ( + output_dir.parent / "example_data" / "Basic_example_vuegen_demo_notebook" + ).resolve() + # copy example data to vuegen_gen folder in home directory + if not path_to_example_data.exists(): + shutil.copytree( + path_to_data_in_bundle, + path_to_example_data, + # dirs_exist_ok=True, + ) + messagebox.showinfo( + "Info", + f"Example data copied to {path_to_example_data}", + ) logo_path = os.path.join(sys._MEIPASS, "vuegen_logo.png") elif app_path.parent.name == "gui": # should be always the case for GUI run from command line - path_to_dat = ( + path_to_example_data = ( app_path.parent.parent / "docs" / "example_data" @@ -87,7 +102,7 @@ app_path.parent.parent / "docs" / "images" / "vuegen_logo.png" ) # 1000x852 pixels else: - path_to_dat = "docs/example_data/Basic_example_vuegen_demo_notebook" + path_to_example_data = "docs/example_data/Basic_example_vuegen_demo_notebook" print(f"{_PATH = }") ########################################################################################## @@ -237,7 +252,7 @@ def select_directory(): ) ctk_radio_config_1.grid(row=row_count, column=1, padx=20, pady=2) -config_path = tk.StringVar(value=str(path_to_dat)) +config_path = tk.StringVar(value=str(path_to_example_data)) config_path_entry = customtkinter.CTkEntry( app, width=400, From 189bcffe3051d22a4de157c2eadf63ee06f7ab40 Mon Sep 17 00:00:00 2001 From: enryh Date: Tue, 18 Mar 2025 09:17:23 +0100 Subject: [PATCH 83/91] :art: remove comment and sort --- src/vuegen/quarto_reportview.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/vuegen/quarto_reportview.py b/src/vuegen/quarto_reportview.py index b5886ce..b0aee2c 100644 --- a/src/vuegen/quarto_reportview.py +++ b/src/vuegen/quarto_reportview.py @@ -870,10 +870,10 @@ def _generate_component_imports(self, component: r.Component) -> List[str]: r.PlotType.PLOTLY: ["import plotly.io as pio", "import requests"], }, "dataframe": [ - "init_notebook_mode(all_interactive=True)", # ! somehow order is random in qmd file "import pandas as pd", "from itables import show, init_notebook_mode", "import dataframe_image as dfi", + "init_notebook_mode(all_interactive=True)", ], "markdown": ["import IPython.display as display", "import requests"], } From ecf8d4ea015ae85fdc6dd0cc09f34362889c7894 Mon Sep 17 00:00:00 2001 From: enryh Date: Tue, 18 Mar 2025 09:19:26 +0100 Subject: [PATCH 84/91] :bug: use selected logger, not always root to reset handlers - handlers were not reset if not root logger was used --- src/vuegen/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/vuegen/utils.py b/src/vuegen/utils.py index e79a9b2..d792b95 100644 --- a/src/vuegen/utils.py +++ b/src/vuegen/utils.py @@ -698,7 +698,7 @@ def init_log( # ! logging.basicConfig has no effect if called once anywhere in the code # ! set handlers and format for the logger manually # Reset any existing handlers - for handler in logging.root.handlers[:]: + for handler in logger.handlers[:]: logger.removeHandler(handler) # Set up the new handlers and format From b8df80aa154f053359c0625949338bb489972493 Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Tue, 18 Mar 2025 13:20:24 +0100 Subject: [PATCH 85/91] :sparkles: document and restore all actions, use windowed build --- .github/workflows/cdci.yml | 22 +++++------ gui/README.md | 76 ++++++++++++++++++++++++++++++++------ 2 files changed, 74 insertions(+), 24 deletions(-) diff --git a/.github/workflows/cdci.yml b/.github/workflows/cdci.yml index dc6704c..3094e18 100644 --- a/.github/workflows/cdci.yml +++ b/.github/workflows/cdci.yml @@ -14,8 +14,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - # python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] - python-version: ["3.11"] + python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] steps: - uses: actions/checkout@v4 - uses: psf/black@stable @@ -44,8 +43,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - # python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] - python-version: ["3.11"] + python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] steps: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} @@ -126,18 +124,18 @@ jobs: build-executable: name: Build-exe-${{ matrix.os.label }} runs-on: ${{ matrix.os.runner }} - # needs: - # - test - # - other-reports + needs: + - test + - other-reports strategy: matrix: python-version: ["3.12"] os: # https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/running-variations-of-jobs-in-a-workflow#example-using-a-multi-dimension-matrix - # - runner: "ubuntu-latest" - # label: "ubuntu-latest_LTS-x64" - # - runner: "macos-13" - # label: "macos-13-x64" + - runner: "ubuntu-latest" + label: "ubuntu-latest_LTS-x64" + - runner: "macos-13" + label: "macos-13-x64" - runner: "macos-15" label: "macos-15-arm64" - runner: "windows-latest" @@ -157,7 +155,7 @@ jobs: - name: Build executable run: | cd gui - pyinstaller -n vuegen_gui --collect-all pyvis --collect-all streamlit --collect-all st_aggrid --collect-all customtkinter --collect-all quarto_cli --collect-all plotly --collect-all _plotly_utils --collect-all traitlets --collect-all referencing --collect-all rpds --collect-all tenacity --collect-all pyvis --collect-all pandas --collect-all numpy --collect-all matplotlib --collect-all openpyxl --collect-all xlrd --collect-all nbformat --collect-all nbclient --collect-all altair --collect-all itables --collect-all kaleido --collect-all pyarrow --collect-all dataframe_image --collect-all narwhals --collect-all PIL --collect-all vl_convert --add-data ../docs/example_data/Basic_example_vuegen_demo_notebook:example_data/Basic_example_vuegen_demo_notebook --add-data ../docs/images/vuegen_logo.png:. app.py + pyinstaller -n vuegen_gui --windowed --collect-all pyvis --collect-all streamlit --collect-all st_aggrid --collect-all customtkinter --collect-all quarto_cli --collect-all plotly --collect-all _plotly_utils --collect-all traitlets --collect-all referencing --collect-all rpds --collect-all tenacity --collect-all pyvis --collect-all pandas --collect-all numpy --collect-all matplotlib --collect-all openpyxl --collect-all xlrd --collect-all nbformat --collect-all nbclient --collect-all altair --collect-all itables --collect-all kaleido --collect-all pyarrow --collect-all dataframe_image --collect-all narwhals --collect-all PIL --collect-all vl_convert --add-data ../docs/example_data/Basic_example_vuegen_demo_notebook:example_data/Basic_example_vuegen_demo_notebook --add-data ../docs/images/vuegen_logo.png:. app.py # --windowed only for mac, see: # https://pyinstaller.org/en/stable/usage.html#building-macos-app-bundles # 'Under macOS, PyInstaller always builds a UNIX executable in dist.' diff --git a/gui/README.md b/gui/README.md index bd486b7..3eaccd7 100644 --- a/gui/README.md +++ b/gui/README.md @@ -67,6 +67,18 @@ Windows and macOS specific options: ## test shipping a python virtual environment with vuegen installed +- [ ] can we ship a python environment with the app which can be used to launch a kernel? + +## Features of the GUI + +- select a directory via a file dialog button +- specify the distination of a config file manually +- select a report +- select if streamlit app should be started - has no effect for quarto reports +- show set PATH +- select a Python environment for starting jupyter kernels for quarto reports which is cached +- some message boxes + ## Bundled PyInstaller execution (current status) 1. Can be executed. Streamlit apps can be run (although sometimes not easily terminated) @@ -78,16 +90,10 @@ Windows and macOS specific options: ## Using bundle vuegen release -## On Windows - -- global quarto and python installations can be used -- quarto can be shipped with app, but maybe it can be deactivated - -## On MacOs - -- on MacOs the default paths are not set +This should both work on Windows and MacOs, but the paths for environments can be different +dependent on the system. -#### Create environment using conda +### Create environment using conda ```bash conda create -n vuegen_gui -c conda-forge python=3.12 jupyter @@ -106,12 +112,58 @@ In the app, set the python environment path to this location, but to the `bin` f /Users/user/miniforge3/envs/vuegen_gui/bin ``` -#### virtualenv +### virtualenv -- tbc +Following the +[Python Packaging User Guide's instructions](https://packaging.python.org/en/latest/guides/installing-using-pip-and-virtual-environments/#create-a-new-virtual-environment) +you can run the following command to create a new virtual environment. -[virutalenv documentation](https://docs.python.org/3/library/venv.html) +Install an offical Python version from [python.org/downloads/](https://www.python.org/downloads/) + +#### On MacOs ```bash +# being in the folder you want to create the environment +python -m venv .venv +# if that does not work, try +# python3 -m venv .venv +source .venv/bin/activate +pip install jupyter +``` + +#### On Windows + +```powershell +# being in the folder you want to create the environment +python -m venv .venv +# if that does not work, try +# py -m venv .venv +.venv\Scripts\activate +``` + +#### Troubleshooting venv + +For more information on the options, see also the +[virutalenv documentation](https://docs.python.org/3/library/venv.html) in the Python +standard library documentation. + +``` python -m venv .venv --copies --clear --prompt vuegenvenv ``` + +### On Windows + +On windows the default Paths is available in the application. This would allow to use +the default python installation and a global quarto installation. + +to test, one could + +- use global quarto and python installations can be used +- add a deactivate button into app for bundled quarto (so path is not set) + +### On MacOs + +- on MacOs the default paths are not set, but only the minimal one `/usr/bin:/bin:/usr/sbin:/sbin`, + see pyinstaller hints + [on path manipulations](https://pyinstaller.org/en/stable/common-issues-and-pitfalls.html#macos) +- requires to add path to python environment manually From da8bc30e8e79854d713e1da08aa4413d3e99004f Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Tue, 18 Mar 2025 15:47:15 +0100 Subject: [PATCH 86/91] :bug: add 'save' tag to ensure that altair plots can be exported --- pyproject.toml | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index d4a8cb8..ee1284a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,13 +17,13 @@ streamlit-aggrid = "*" quarto-cli = "*" plotly = "5.15.0" pyvis = "^0.3.2" -pandas = {extras = ["parquet"], version = "^2.2.3"} +pandas = { extras = ["parquet"], version = "^2.2.3" } openpyxl = "^3.1.5" xlrd = "^2.0.1" nbformat = "^5.10.4" nbclient = "^0.10.0" matplotlib = "^3.9.2" -altair = "*" +altair = { extras = ["save"], version = "*" } itables = "^2.2.2" kaleido = "0.2.0" vl-convert-python = "^1.7.0" @@ -33,20 +33,20 @@ pyyaml = "^6.0.2" # optional doc depencencies, follow approach as described here: # https://github.com/python-poetry/poetry/issues/2567#issuecomment-646766059 -sphinx = {version="*", optional=true} -sphinx-book-theme = {version="*", optional=true} -myst-nb = {version="*", optional=true} -ipywidgets = {version="*", optional=true} -sphinx-new-tab-link = {version = "!=0.2.2", optional=true} -jupytext = {version="*", optional=true} -customtkinter = {version="*", optional=true} +sphinx = { version = "*", optional = true } +sphinx-book-theme = { version = "*", optional = true } +myst-nb = { version = "*", optional = true } +ipywidgets = { version = "*", optional = true } +sphinx-new-tab-link = { version = "!=0.2.2", optional = true } +jupytext = { version = "*", optional = true } +customtkinter = { version = "*", optional = true } [tool.poetry.group.dev.dependencies] -ipykernel = {version="^6.29.5", optional=true} +ipykernel = { version = "^6.29.5", optional = true } [tool.poetry.requires-plugins] poetry-dynamic-versioning = { version = ">=1.0.0,<2.0.0", extras = ["plugin"] } - + [tool.poetry-dynamic-versioning] enable = true @@ -56,7 +56,14 @@ build-backend = "poetry_dynamic_versioning.backend" # https://stackoverflow.com/a/60990574/9684872 [tool.poetry.extras] -docs = ["sphinx", "sphinx-book-theme", "myst-nb", "ipywidgets", "sphinx-new-tab-link", "jupytext"] +docs = [ + "sphinx", + "sphinx-book-theme", + "myst-nb", + "ipywidgets", + "sphinx-new-tab-link", + "jupytext", +] gui = ["customtkinter"] [tool.poetry.scripts] From e88a93b9231e16834b3bf39a0b013a91560612fd Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Tue, 18 Mar 2025 16:02:16 +0100 Subject: [PATCH 87/91] :art: format docs as they were adapted - api changes needed to be reflected in docs. - format docs - enforce using action --- .github/workflows/cdci.yml | 2 + docs/vuegen_basic_case_study.ipynb | 77 +- docs/vuegen_case_study_earth_microbiome.ipynb | 726 +++++++++++------- 3 files changed, 500 insertions(+), 305 deletions(-) diff --git a/.github/workflows/cdci.yml b/.github/workflows/cdci.yml index 3094e18..8bfa271 100644 --- a/.github/workflows/cdci.yml +++ b/.github/workflows/cdci.yml @@ -18,6 +18,8 @@ jobs: steps: - uses: actions/checkout@v4 - uses: psf/black@stable + with: + jupyter: true - uses: isort/isort-action@v1 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 diff --git a/docs/vuegen_basic_case_study.ipynb b/docs/vuegen_basic_case_study.ipynb index ff3218b..834b29f 100644 --- a/docs/vuegen_basic_case_study.ipynb +++ b/docs/vuegen_basic_case_study.ipynb @@ -67,7 +67,7 @@ }, "outputs": [], "source": [ - "# Vuegen library \n", + "# Vuegen library\n", "%pip install vuegen" ] }, @@ -78,6 +78,7 @@ "outputs": [], "source": [ "import os\n", + "\n", "IN_COLAB = \"COLAB_GPU\" in os.environ" ] }, @@ -181,14 +182,20 @@ "# run_streamlit = True # uncomment line to run the streamlit report\n", "# Launch the Streamlit report depneding on the platform\n", "if not IN_COLAB and run_streamlit:\n", - " !streamlit run streamlit_report/sections/report_manager.py\n", + " !streamlit run streamlit_report/sections/report_manager.py\n", "elif run_streamlit:\n", - " # see: https://discuss.streamlit.io/t/how-to-launch-streamlit-app-from-google-colab-notebook/42399\n", - " print(\"Password/Enpoint IP for localtunnel is:\",urllib.request.urlopen('https://ipv4.icanhazip.com').read().decode('utf8').strip(\"\\n\"))\n", - " # Run the Streamlit app in the background\n", - " !streamlit run streamlit_report/sections/report_manager.py --server.address=localhost &>/content/logs.txt &\n", - " # Expose the Streamlit app on port 8501\n", - " !npx localtunnel --port 8501 --subdomain vuegen-demo\n", + " # see: https://discuss.streamlit.io/t/how-to-launch-streamlit-app-from-google-colab-notebook/42399\n", + " print(\n", + " \"Password/Enpoint IP for localtunnel is:\",\n", + " urllib.request.urlopen(\"https://ipv4.icanhazip.com\")\n", + " .read()\n", + " .decode(\"utf8\")\n", + " .strip(\"\\n\"),\n", + " )\n", + " # Run the Streamlit app in the background\n", + " !streamlit run streamlit_report/sections/report_manager.py --server.address=localhost &>/content/logs.txt &\n", + " # Expose the Streamlit app on port 8501\n", + " !npx localtunnel --port 8501 --subdomain vuegen-demo\n", "else:\n", " print(\"Streamlit report not executed, set run_streamlit to True to run the report\")" ] @@ -208,7 +215,9 @@ "source": [ "# Generate the report\n", "report_type = \"html\"\n", - "report_dir, config_path = report_generator.get_report(dir_path = base_output_dir, report_type = report_type, logger = None)\n", + "report_dir, config_path = report_generator.get_report(\n", + " dir_path=base_output_dir, report_type=report_type, logger=None\n", + ")\n", "print(f\"Report generated at: {report_dir}\")" ] }, @@ -236,11 +245,15 @@ "vuegen_logo_path = \"https://raw.githubusercontent.com/Multiomics-Analytics-Group/vuegen/main/docs/images/vuegen_logo.svg\"\n", "\n", "# Load the YAML file\n", - "print(f\"Loading the YAML config file from: {config_path}\") # generated based on directory path above\n", + "print(\n", + " f\"Loading the YAML config file from: {config_path}\"\n", + ") # generated based on directory path above\n", "config = load_yaml_config(config_path)\n", "\n", "# Update the logo and graphical abstract with the URL\n", - "config[\"report\"].update({\"logo\": vuegen_logo_path, \"graphical_abstract\": vuegen_logo_path})" + "config[\"report\"].update(\n", + " {\"logo\": vuegen_logo_path, \"graphical_abstract\": vuegen_logo_path}\n", + ")" ] }, { @@ -258,7 +271,7 @@ "source": [ "# Update the description for the EDA section\n", "for section in config[\"sections\"]:\n", - " if section[\"title\"] == \"Plots\": \n", + " if section[\"title\"] == \"Plots\":\n", " section[\"description\"] = \"This section contains example plots\"\n", "\n", "# Update the description for the alpha diversity subsection from the Metagenomics section\n", @@ -266,7 +279,9 @@ " if section[\"title\"] == \"Dataframes\":\n", " for subsection in section[\"subsections\"]:\n", " if subsection[\"title\"] == \"All Formats\":\n", - " subsection[\"description\"] = \"This subsection contains example dataframes.\"\n" + " subsection[\"description\"] = (\n", + " \"This subsection contains example dataframes.\"\n", + " )" ] }, { @@ -285,11 +300,11 @@ "# Define new plot with a URL as the file path\n", "vuegen_abst_fig = {\n", " \"title\": \"Graphical overview of VueGen’s workflow and components\",\n", - " \"file_path\": \"https://raw.githubusercontent.com/Multiomics-Analytics-Group/vuegen/main/docs/images/vuegen_graph_abstract.png\", \n", + " \"file_path\": \"https://raw.githubusercontent.com/Multiomics-Analytics-Group/vuegen/main/docs/images/vuegen_graph_abstract.png\",\n", " \"description\": \"\",\n", " \"caption\": \"The diagram illustrates the processing pipeline of VueGen, starting from either a directory or a YAML configuration file. Reports consist of hierarchical sections and subsections, each containing various components such as plots, dataframes, Markdown, HTML, and data retrieved via API calls.\",\n", " \"component_type\": \"plot\",\n", - " \"plot_type\": \"static\"\n", + " \"plot_type\": \"static\",\n", "}\n", "\n", "# Add the plot to the Sample Provenance subsection in the EDA section\n", @@ -321,7 +336,9 @@ "source": [ "# Test the changes by generarating the report from the modified YAML file\n", "report_type = \"streamlit\"\n", - "_ = report_generator.get_report(config_path = config_path, report_type = report_type, logger = None)" + "_ = report_generator.get_report(\n", + " config_path=config_path, report_type=report_type, logger=None\n", + ")" ] }, { @@ -334,14 +351,20 @@ "# run_streamlit = True # uncomment line to run the streamlit report\n", "# Launch the Streamlit report depneding on the platform\n", "if not IN_COLAB and run_streamlit:\n", - " !streamlit run streamlit_report/sections/report_manager.py\n", + " !streamlit run streamlit_report/sections/report_manager.py\n", "elif run_streamlit:\n", - " # see: https://discuss.streamlit.io/t/how-to-launch-streamlit-app-from-google-colab-notebook/42399\n", - " print(\"Password/Enpoint IP for localtunnel is:\",urllib.request.urlopen('https://ipv4.icanhazip.com').read().decode('utf8').strip(\"\\n\"))\n", - " # Run the Streamlit app in the background\n", - " !streamlit run streamlit_report/sections/report_manager.py --server.address=localhost &>/content/logs.txt &\n", - " # Expose the Streamlit app on port 8501\n", - " !npx localtunnel --port 8501 --subdomain vuegen-demo\n", + " # see: https://discuss.streamlit.io/t/how-to-launch-streamlit-app-from-google-colab-notebook/42399\n", + " print(\n", + " \"Password/Enpoint IP for localtunnel is:\",\n", + " urllib.request.urlopen(\"https://ipv4.icanhazip.com\")\n", + " .read()\n", + " .decode(\"utf8\")\n", + " .strip(\"\\n\"),\n", + " )\n", + " # Run the Streamlit app in the background\n", + " !streamlit run streamlit_report/sections/report_manager.py --server.address=localhost &>/content/logs.txt &\n", + " # Expose the Streamlit app on port 8501\n", + " !npx localtunnel --port 8501 --subdomain vuegen-demo\n", "else:\n", " print(\"Streamlit report not executed, set run_streamlit to True to run the report\")" ] @@ -361,13 +384,15 @@ "source": [ "# Test the changes by generarating the report from the modified YAML file\n", "report_type = \"html\"\n", - "_ = report_generator.get_report(config_path = config_path, report_type = report_type, logger = None)" + "_ = report_generator.get_report(\n", + " config_path=config_path, report_type=report_type, logger=None\n", + ")" ] } ], "metadata": { "kernelspec": { - "display_name": "vuegen", + "display_name": "vuegen_py312", "language": "python", "name": "python3" }, @@ -381,7 +406,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.21" + "version": "3.12.9" } }, "nbformat": 4, diff --git a/docs/vuegen_case_study_earth_microbiome.ipynb b/docs/vuegen_case_study_earth_microbiome.ipynb index b9ddad4..70d81b2 100644 --- a/docs/vuegen_case_study_earth_microbiome.ipynb +++ b/docs/vuegen_case_study_earth_microbiome.ipynb @@ -70,7 +70,7 @@ "metadata": {}, "outputs": [], "source": [ - "# Vuegen library \n", + "# Vuegen library\n", "%pip install vuegen" ] }, @@ -91,6 +91,7 @@ "outputs": [], "source": [ "import os\n", + "\n", "IN_COLAB = \"COLAB_GPU\" in os.environ" ] }, @@ -216,42 +217,43 @@ " input:\n", " empocat- empo category name (string)\n", " returndict- the user needs the dictionary mapping category to color (boolean)\n", - " \n", + "\n", " output: either a color for passed empocat or the dictionay if returndict=True\"\"\"\n", - " \n", + "\n", " # hex codes for matplotlib colors are described here:\n", " # https://github.com/matplotlib/matplotlib/blob/cf83cd5642506ef808853648b9eb409f8dbd6ff3/lib/matplotlib/_color_data.py\n", "\n", - " empo_cat_color={'EMP sample': '#929591', # 'grey'\n", - " 'Host-associated': '#fb9a99',\n", - " 'Free-living': '#e31a1c',\n", - " 'Animal': '#b2df8a',\n", - " 'Plant': '#33a02c',\n", - " 'Non-saline': '#a6cee3',\n", - " 'Saline': '#1f78b4',\n", - " 'Aerosol (non-saline)': '#d3d3d3', # 'lightgrey'\n", - " 'Animal corpus': '#ffff00', # 'yellow'\n", - " 'Animal distal gut': '#8b4513', # 'saddlebrown'\n", - " 'Animal proximal gut': '#d2b48c', # 'tan'\n", - " 'Animal secretion': '#f4a460', # 'sandybrown'\n", - " 'Animal surface': '#b8860b', # 'darkgoldenrod'\n", - " 'Hypersaline (saline)': '#87cefa', # 'lightskyblue'\n", - " 'Intertidal (saline)': '#afeeee', # 'paleturquoise'\n", - " 'Mock community': '#ff00ff', # 'fuchsia'\n", - " 'Plant corpus': '#7cfc00', # 'lawngreen'\n", - " 'Plant rhizosphere': '#006400', # 'darkgreen'\n", - " 'Plant surface': '#00fa9a', # 'mediumspringgreen'\n", - " 'Sediment (non-saline)': '#ffa07a', # 'lightsalmon'\n", - " 'Sediment (saline)': '#ff6347', # 'tomato'\n", - " 'Soil (non-saline)': '#ff0000', # 'red'\n", - " 'Sterile water blank': '#ee82ee', # 'violet'\n", - " 'Surface (non-saline)': '#000000', # 'black'\n", - " 'Surface (saline)': '#696969', # 'dimgrey'\n", - " 'Water (non-saline)': '#000080', # 'navy'\n", - " 'Water (saline)': '#4169e1' # 'royalblue'\n", - " }\n", - " \n", - " if returndict==True:\n", + " empo_cat_color = {\n", + " \"EMP sample\": \"#929591\", # 'grey'\n", + " \"Host-associated\": \"#fb9a99\",\n", + " \"Free-living\": \"#e31a1c\",\n", + " \"Animal\": \"#b2df8a\",\n", + " \"Plant\": \"#33a02c\",\n", + " \"Non-saline\": \"#a6cee3\",\n", + " \"Saline\": \"#1f78b4\",\n", + " \"Aerosol (non-saline)\": \"#d3d3d3\", # 'lightgrey'\n", + " \"Animal corpus\": \"#ffff00\", # 'yellow'\n", + " \"Animal distal gut\": \"#8b4513\", # 'saddlebrown'\n", + " \"Animal proximal gut\": \"#d2b48c\", # 'tan'\n", + " \"Animal secretion\": \"#f4a460\", # 'sandybrown'\n", + " \"Animal surface\": \"#b8860b\", # 'darkgoldenrod'\n", + " \"Hypersaline (saline)\": \"#87cefa\", # 'lightskyblue'\n", + " \"Intertidal (saline)\": \"#afeeee\", # 'paleturquoise'\n", + " \"Mock community\": \"#ff00ff\", # 'fuchsia'\n", + " \"Plant corpus\": \"#7cfc00\", # 'lawngreen'\n", + " \"Plant rhizosphere\": \"#006400\", # 'darkgreen'\n", + " \"Plant surface\": \"#00fa9a\", # 'mediumspringgreen'\n", + " \"Sediment (non-saline)\": \"#ffa07a\", # 'lightsalmon'\n", + " \"Sediment (saline)\": \"#ff6347\", # 'tomato'\n", + " \"Soil (non-saline)\": \"#ff0000\", # 'red'\n", + " \"Sterile water blank\": \"#ee82ee\", # 'violet'\n", + " \"Surface (non-saline)\": \"#000000\", # 'black'\n", + " \"Surface (saline)\": \"#696969\", # 'dimgrey'\n", + " \"Water (non-saline)\": \"#000080\", # 'navy'\n", + " \"Water (saline)\": \"#4169e1\", # 'royalblue'\n", + " }\n", + "\n", + " if returndict == True:\n", " return empo_cat_color\n", " else:\n", " return empo_cat_color[empocat]" @@ -286,13 +288,15 @@ "outputs": [], "source": [ "# Create the output directory for the EDA section and sample provenance subsection\n", - "sample_prov_output_dir = os.path.join(base_output_dir, \"1_Exploratory_data_analysis/1_sample_exploration/\")\n", + "sample_prov_output_dir = os.path.join(\n", + " base_output_dir, \"1_Exploratory_data_analysis/1_sample_exploration/\"\n", + ")\n", "os.makedirs(sample_prov_output_dir, exist_ok=True)\n", "\n", "# Load data and filter out control samples\n", - "metadata_mapping = 'https://raw.githubusercontent.com//biocore/emp/master/data/mapping-files/emp_qiime_mapping_release1.tsv'\n", + "metadata_mapping = \"https://raw.githubusercontent.com//biocore/emp/master/data/mapping-files/emp_qiime_mapping_release1.tsv\"\n", "metadata_mapping_df = pd.read_table(metadata_mapping, index_col=0)\n", - "metadata_mapping_df = metadata_mapping_df[metadata_mapping_df['empo_1'] != 'Control']" + "metadata_mapping_df = metadata_mapping_df[metadata_mapping_df[\"empo_1\"] != \"Control\"]" ] }, { @@ -312,7 +316,9 @@ "sample_metadata_mapping_df = metadata_mapping_df.sample(100, random_state=42)\n", "\n", "# Export the sample df as a CSV file\n", - "sample_metadata_mapping_df.to_csv(f'{sample_prov_output_dir}/1_metadata_random_subset.csv')" + "sample_metadata_mapping_df.to_csv(\n", + " f\"{sample_prov_output_dir}/1_metadata_random_subset.csv\"\n", + ")" ] }, { @@ -335,12 +341,14 @@ "animal_empo3 = animal_df[\"empo_3\"].unique()\n", "\n", "# Create a figure with Cartopy map projection\n", - "fig, ax = plt.subplots(figsize=(12, 8), dpi=300, subplot_kw={'projection': ccrs.PlateCarree()})\n", + "fig, ax = plt.subplots(\n", + " figsize=(12, 8), dpi=300, subplot_kw={\"projection\": ccrs.PlateCarree()}\n", + ")\n", "\n", "# Add features to the map\n", - "ax.add_feature(cfeature.BORDERS, edgecolor='white', linewidth=0.5)\n", - "ax.add_feature(cfeature.LAND, edgecolor='white', facecolor='lightgray', linewidth=0.5)\n", - "ax.add_feature(cfeature.COASTLINE, edgecolor='white', linewidth=0.5)\n", + "ax.add_feature(cfeature.BORDERS, edgecolor=\"white\", linewidth=0.5)\n", + "ax.add_feature(cfeature.LAND, edgecolor=\"white\", facecolor=\"lightgray\", linewidth=0.5)\n", + "ax.add_feature(cfeature.COASTLINE, edgecolor=\"white\", linewidth=0.5)\n", "\n", "# Set extent (global map)\n", "ax.set_extent([-180, 180, -90, 90])\n", @@ -349,13 +357,21 @@ "for empo3 in animal_empo3:\n", " subset = animal_df[animal_df[\"empo_3\"] == empo3]\n", " color = get_empo_cat_color(empo3) # Get color for category\n", - " ax.scatter(subset[\"longitude_deg\"], subset[\"latitude_deg\"], \n", - " color='none', edgecolors=color, linewidth=1.5, label=empo3, s=40, \n", - " transform=ccrs.PlateCarree(), zorder=2)\n", + " ax.scatter(\n", + " subset[\"longitude_deg\"],\n", + " subset[\"latitude_deg\"],\n", + " color=\"none\",\n", + " edgecolors=color,\n", + " linewidth=1.5,\n", + " label=empo3,\n", + " s=40,\n", + " transform=ccrs.PlateCarree(),\n", + " zorder=2,\n", + " )\n", "\n", "# Add legend with updated labels\n", "handles, labels = ax.get_legend_handles_labels()\n", - "ax.legend(handles, labels, loc='lower center', ncol=2, fontsize=10)\n", + "ax.legend(handles, labels, loc=\"lower center\", ncol=2, fontsize=10)\n", "\n", "# Save the figure\n", "animal_map_out_path = os.path.join(sample_prov_output_dir, \"2_animal_samples_map.png\")\n", @@ -376,7 +392,7 @@ "outputs": [], "source": [ "# Extract Plant dataset\n", - "plant_df = metadata_mapping_df[metadata_mapping_df['empo_2'] == 'Plant']\n", + "plant_df = metadata_mapping_df[metadata_mapping_df[\"empo_2\"] == \"Plant\"]\n", "\n", "# Unique subcategories in empo_3\n", "plant_empo3 = plant_df[\"empo_3\"].unique()\n", @@ -389,18 +405,22 @@ " subset = plant_df[plant_df[\"empo_3\"] == empo3]\n", " color = get_empo_cat_color(empo3) # Get color for category\n", "\n", - " fig.add_trace(go.Scattergeo(\n", - " lon=subset[\"longitude_deg\"],\n", - " lat=subset[\"latitude_deg\"],\n", - " mode=\"markers\",\n", - " marker=dict(\n", - " symbol=\"circle-open\", # Unfilled circle\n", - " color=color,\n", - " size=6, # Marker size\n", - " line=dict(width=1.5, color=color) # Border color matches category color\n", - " ),\n", - " name=empo3\n", - " ))\n", + " fig.add_trace(\n", + " go.Scattergeo(\n", + " lon=subset[\"longitude_deg\"],\n", + " lat=subset[\"latitude_deg\"],\n", + " mode=\"markers\",\n", + " marker=dict(\n", + " symbol=\"circle-open\", # Unfilled circle\n", + " color=color,\n", + " size=6, # Marker size\n", + " line=dict(\n", + " width=1.5, color=color\n", + " ), # Border color matches category color\n", + " ),\n", + " name=empo3,\n", + " )\n", + " )\n", "\n", "# Update map layout (fixes horizontal blank space)\n", "fig.update_layout(\n", @@ -412,7 +432,7 @@ " coastlinecolor=\"white\",\n", " fitbounds=\"locations\", # Focuses only on data points\n", " lataxis=dict(range=[-60, 85], showgrid=False), # Custom latitude range\n", - " lonaxis=dict(range=[-180, 180], showgrid=False) # Custom longitude range\n", + " lonaxis=dict(range=[-180, 180], showgrid=False), # Custom longitude range\n", " ),\n", " autosize=False,\n", " width=800, # Adjust width to remove blank space\n", @@ -424,8 +444,8 @@ " x=0.5, # Center legend horizontally\n", " xanchor=\"center\",\n", " yanchor=\"top\",\n", - " orientation=\"h\" # Horizontal legend layout\n", - " )\n", + " orientation=\"h\", # Horizontal legend layout\n", + " ),\n", ")\n", "\n", "# Save the figure as SVG and JSON\n", @@ -457,50 +477,62 @@ "\n", "# Create a dictionary for simplified category names for the legend\n", "simplified_category_names = {\n", - " 'Water (saline)': 'Water',\n", - " 'Sediment (saline)': 'Sediment',\n", - " 'Surface (saline)': 'Surface',\n", - " 'Hypersaline (saline)': 'Hypersaline'\n", + " \"Water (saline)\": \"Water\",\n", + " \"Sediment (saline)\": \"Sediment\",\n", + " \"Surface (saline)\": \"Surface\",\n", + " \"Hypersaline (saline)\": \"Hypersaline\",\n", "}\n", "\n", "# Simplify the empo_3 names in the DataFrame for legend\n", - "saline_df['simplified_empo_3'] = saline_df['empo_3'].apply(lambda x: simplified_category_names.get(x, x))\n", + "saline_df[\"simplified_empo_3\"] = saline_df[\"empo_3\"].apply(\n", + " lambda x: simplified_category_names.get(x, x)\n", + ")\n", "\n", "# Apply the get_empo_cat_color function to generate the color column\n", - "saline_df['color'] = saline_df['empo_3'].apply(get_empo_cat_color)\n", + "saline_df[\"color\"] = saline_df[\"empo_3\"].apply(get_empo_cat_color)\n", "\n", "# Create the base world map (use the CDN URL or a different base map if you prefer)\n", - "countries = alt.topo_feature('https://cdn.jsdelivr.net/npm/world-atlas@2/countries-110m.json', 'countries')\n", + "countries = alt.topo_feature(\n", + " \"https://cdn.jsdelivr.net/npm/world-atlas@2/countries-110m.json\", \"countries\"\n", + ")\n", "\n", "# Create a background map of countries\n", - "background_map = alt.Chart(countries).mark_geoshape(\n", - " fill='lightgray',\n", - " stroke='white'\n", - ").project('equirectangular').properties(\n", - " width=800,\n", - " height=400\n", + "background_map = (\n", + " alt.Chart(countries)\n", + " .mark_geoshape(fill=\"lightgray\", stroke=\"white\")\n", + " .project(\"equirectangular\")\n", + " .properties(width=800, height=400)\n", ")\n", "\n", "# Create the points for saline samples with custom colors\n", - "saline_points = alt.Chart(saline_df).mark_point(size=50, shape='circle', filled=False,).encode(\n", - " longitude='longitude_deg:Q',\n", - " latitude='latitude_deg:Q',\n", - " color=alt.Color('simplified_empo_3:N', \n", - " scale=alt.Scale(domain=list(saline_df['simplified_empo_3'].unique()), \n", - " range=[get_empo_cat_color(cat) for cat in saline_df['empo_3'].unique()]),\n", - " legend=alt.Legend(\n", - " title='',\n", - " orient='bottom', \n", - " symbolSize=120,\n", - " labelFontSize=16 \n", - " )),\n", - " tooltip=['latitude_deg', 'longitude_deg', 'empo_3']\n", + "saline_points = (\n", + " alt.Chart(saline_df)\n", + " .mark_point(\n", + " size=50,\n", + " shape=\"circle\",\n", + " filled=False,\n", + " )\n", + " .encode(\n", + " longitude=\"longitude_deg:Q\",\n", + " latitude=\"latitude_deg:Q\",\n", + " color=alt.Color(\n", + " \"simplified_empo_3:N\",\n", + " scale=alt.Scale(\n", + " domain=list(saline_df[\"simplified_empo_3\"].unique()),\n", + " range=[get_empo_cat_color(cat) for cat in saline_df[\"empo_3\"].unique()],\n", + " ),\n", + " legend=alt.Legend(\n", + " title=\"\", orient=\"bottom\", symbolSize=120, labelFontSize=16\n", + " ),\n", + " ),\n", + " tooltip=[\"latitude_deg\", \"longitude_deg\", \"empo_3\"],\n", + " )\n", ")\n", "\n", "# Overlay the points on the world map\n", "map_with_points = background_map + saline_points\n", "\n", - "# Save as JSON and \n", + "# Save as JSON and\n", "saline_json_path = os.path.join(sample_prov_output_dir, \"4_saline_samples_map.json\")\n", "with open(saline_json_path, \"w\") as f:\n", " f.write(map_with_points.to_json())\n", @@ -544,9 +576,11 @@ "alpha_div_output_dir = os.path.join(base_output_dir, \"2_Metagenomics/1_alpha_diversity\")\n", "os.makedirs(alpha_div_output_dir, exist_ok=True)\n", "\n", - "# Load data \n", - "mapping_qc_filt = 'https://raw.githubusercontent.com//biocore/emp/master/data/mapping-files/emp_qiime_mapping_qc_filtered.tsv'\n", - "mapping_qc_filt_df = pd.read_csv(mapping_qc_filt, sep='\\t', index_col=0, header=0).sort_index()" + "# Load data\n", + "mapping_qc_filt = \"https://raw.githubusercontent.com//biocore/emp/master/data/mapping-files/emp_qiime_mapping_qc_filtered.tsv\"\n", + "mapping_qc_filt_df = pd.read_csv(\n", + " mapping_qc_filt, sep=\"\\t\", index_col=0, header=0\n", + ").sort_index()" ] }, { @@ -555,28 +589,28 @@ "metadata": {}, "outputs": [], "source": [ - "# Define colors of host associated and free living categories \n", + "# Define colors of host associated and free living categories\n", "colorsHA = {\n", - " 'Animal corpus': get_empo_cat_color('Animal corpus'),\n", - " 'Plant corpus': get_empo_cat_color('Plant corpus'),\n", - " 'Animal secretion': get_empo_cat_color('Animal secretion'),\n", - " 'Plant surface': get_empo_cat_color('Plant surface'),\n", - " 'Animal proximal gut': get_empo_cat_color('Animal proximal gut'),\n", - " 'Animal surface': get_empo_cat_color('Animal surface'),\n", - " 'Animal distal gut': get_empo_cat_color('Animal distal gut'),\n", - " 'Plant rhizosphere': get_empo_cat_color('Plant rhizosphere'),\n", + " \"Animal corpus\": get_empo_cat_color(\"Animal corpus\"),\n", + " \"Plant corpus\": get_empo_cat_color(\"Plant corpus\"),\n", + " \"Animal secretion\": get_empo_cat_color(\"Animal secretion\"),\n", + " \"Plant surface\": get_empo_cat_color(\"Plant surface\"),\n", + " \"Animal proximal gut\": get_empo_cat_color(\"Animal proximal gut\"),\n", + " \"Animal surface\": get_empo_cat_color(\"Animal surface\"),\n", + " \"Animal distal gut\": get_empo_cat_color(\"Animal distal gut\"),\n", + " \"Plant rhizosphere\": get_empo_cat_color(\"Plant rhizosphere\"),\n", "}\n", "\n", "colorsFL = {\n", - " 'Water (saline)': get_empo_cat_color('Water (saline)'), \n", - " 'Aerosol (non-saline)': get_empo_cat_color('Aerosol (non-saline)'), \n", - " 'Hypersaline (saline)': get_empo_cat_color('Hypersaline (saline)'),\n", - " 'Surface (non-saline)': get_empo_cat_color('Surface (non-saline)'), \n", - " 'Surface (saline)': get_empo_cat_color('Surface (saline)'), \n", - " 'Water (non-saline)': get_empo_cat_color('Water (non-saline)'), \n", - " 'Sediment (saline)': get_empo_cat_color('Sediment (saline)'), \n", - " 'Soil (non-saline)': get_empo_cat_color('Soil (non-saline)'), \n", - " 'Sediment (non-saline)': get_empo_cat_color('Sediment (non-saline)')\n", + " \"Water (saline)\": get_empo_cat_color(\"Water (saline)\"),\n", + " \"Aerosol (non-saline)\": get_empo_cat_color(\"Aerosol (non-saline)\"),\n", + " \"Hypersaline (saline)\": get_empo_cat_color(\"Hypersaline (saline)\"),\n", + " \"Surface (non-saline)\": get_empo_cat_color(\"Surface (non-saline)\"),\n", + " \"Surface (saline)\": get_empo_cat_color(\"Surface (saline)\"),\n", + " \"Water (non-saline)\": get_empo_cat_color(\"Water (non-saline)\"),\n", + " \"Sediment (saline)\": get_empo_cat_color(\"Sediment (saline)\"),\n", + " \"Soil (non-saline)\": get_empo_cat_color(\"Soil (non-saline)\"),\n", + " \"Sediment (non-saline)\": get_empo_cat_color(\"Sediment (non-saline)\"),\n", "}" ] }, @@ -594,20 +628,22 @@ "outputs": [], "source": [ "# Ensure y variable is numeric to avoid aggregation errors\n", - "mapping_qc_filt_df['adiv_observed_otus'] = pd.to_numeric(mapping_qc_filt_df['adiv_observed_otus'], errors='coerce')\n", + "mapping_qc_filt_df[\"adiv_observed_otus\"] = pd.to_numeric(\n", + " mapping_qc_filt_df[\"adiv_observed_otus\"], errors=\"coerce\"\n", + ")\n", "\n", "# Get valid categories (only ones in colorsHA)\n", "valid_categories_HA = set(colorsHA.keys())\n", "\n", "# Filter dataset to include only valid categories\n", "filtered_data_HA = mapping_qc_filt_df[\n", - " (mapping_qc_filt_df['empo_0'] == 'EMP sample') &\n", - " (mapping_qc_filt_df['empo_3'].isin(valid_categories_HA)) \n", + " (mapping_qc_filt_df[\"empo_0\"] == \"EMP sample\")\n", + " & (mapping_qc_filt_df[\"empo_3\"].isin(valid_categories_HA))\n", "]\n", "\n", "# Compute sorted order (only for valid categories)\n", "sorted_order = (\n", - " filtered_data_HA.groupby(['empo_3'])['adiv_observed_otus']\n", + " filtered_data_HA.groupby([\"empo_3\"])[\"adiv_observed_otus\"]\n", " .mean()\n", " .dropna()\n", " .sort_values()\n", @@ -621,21 +657,36 @@ "fig = plt.figure(figsize=(16, 8))\n", "\n", "# Plot the boxplot and jitter plot\n", - "sns.boxplot(fliersize=0, x='empo_3', y='adiv_observed_otus', hue='empo_3', linewidth=1, data=filtered_data_HA, \n", - " order=sorted_order, palette=palette_dict)\n", - "sns.stripplot(jitter=True, x='empo_3', y='adiv_observed_otus', data=filtered_data_HA, order=sorted_order, \n", - " color='black', size=1)\n", + "sns.boxplot(\n", + " fliersize=0,\n", + " x=\"empo_3\",\n", + " y=\"adiv_observed_otus\",\n", + " hue=\"empo_3\",\n", + " linewidth=1,\n", + " data=filtered_data_HA,\n", + " order=sorted_order,\n", + " palette=palette_dict,\n", + ")\n", + "sns.stripplot(\n", + " jitter=True,\n", + " x=\"empo_3\",\n", + " y=\"adiv_observed_otus\",\n", + " data=filtered_data_HA,\n", + " order=sorted_order,\n", + " color=\"black\",\n", + " size=1,\n", + ")\n", "\n", "# Customize the plot\n", - "plt.xticks(rotation=45, ha='right', fontsize=16)\n", + "plt.xticks(rotation=45, ha=\"right\", fontsize=16)\n", "plt.yticks(fontsize=16)\n", - "plt.xlabel('')\n", + "plt.xlabel(\"\")\n", "plt.ylim(0, 3000)\n", - "plt.ylabel('Observed tag sequences', fontsize=16)\n", + "plt.ylabel(\"Observed tag sequences\", fontsize=16)\n", "\n", "# Add median line\n", - "median = filtered_data_HA['adiv_observed_otus'].median()\n", - "plt.axhline(y=median, xmin=0, xmax=1, color='y')\n", + "median = filtered_data_HA[\"adiv_observed_otus\"].median()\n", + "plt.axhline(y=median, xmin=0, xmax=1, color=\"y\")\n", "\n", "# Adjust layout and save the figure\n", "plt.tight_layout()\n", @@ -645,7 +696,9 @@ "os.makedirs(alpha_div_output_dir, exist_ok=True)\n", "\n", "# Save figure\n", - "alpha_div_box_plot_host_ass = os.path.join(alpha_div_output_dir, \"1_alpha_diversity_host_associated_samples.png\")\n", + "alpha_div_box_plot_host_ass = os.path.join(\n", + " alpha_div_output_dir, \"1_alpha_diversity_host_associated_samples.png\"\n", + ")\n", "plt.savefig(alpha_div_box_plot_host_ass, dpi=300, bbox_inches=\"tight\")" ] }, @@ -663,20 +716,22 @@ "outputs": [], "source": [ "# Ensure y variable is numeric to avoid aggregation errors\n", - "mapping_qc_filt_df['adiv_observed_otus'] = pd.to_numeric(mapping_qc_filt_df['adiv_observed_otus'], errors='coerce')\n", + "mapping_qc_filt_df[\"adiv_observed_otus\"] = pd.to_numeric(\n", + " mapping_qc_filt_df[\"adiv_observed_otus\"], errors=\"coerce\"\n", + ")\n", "\n", "# Get valid free-living categories (only ones in colorsFL)\n", "valid_categories_FL = list(colorsFL.keys())\n", "\n", "# Filter dataset to include only valid free-living categories\n", "filtered_data_FL = mapping_qc_filt_df[\n", - " (mapping_qc_filt_df['empo_0'] == 'EMP sample') &\n", - " (mapping_qc_filt_df['empo_3'].isin(valid_categories_FL)) \n", + " (mapping_qc_filt_df[\"empo_0\"] == \"EMP sample\")\n", + " & (mapping_qc_filt_df[\"empo_3\"].isin(valid_categories_FL))\n", "]\n", "\n", "# Compute sorted order (only for valid categories)\n", "sorted_order_FL = (\n", - " filtered_data_FL.groupby(['empo_3'])['adiv_observed_otus']\n", + " filtered_data_FL.groupby([\"empo_3\"])[\"adiv_observed_otus\"]\n", " .mean()\n", " .dropna()\n", " .sort_values()\n", @@ -686,51 +741,44 @@ "# Create the Plotly figure using boxplot and stripplot (jittered points)\n", "fig = px.box(\n", " filtered_data_FL,\n", - " x='empo_3',\n", - " y='adiv_observed_otus',\n", - " color='empo_3',\n", - " category_orders={'empo_3': sorted_order_FL},\n", + " x=\"empo_3\",\n", + " y=\"adiv_observed_otus\",\n", + " color=\"empo_3\",\n", + " category_orders={\"empo_3\": sorted_order_FL},\n", " color_discrete_map=colorsFL,\n", - " labels={'adiv_observed_otus': 'Observed tag sequences'},\n", - " points=False\n", + " labels={\"adiv_observed_otus\": \"Observed tag sequences\"},\n", + " points=False,\n", ")\n", "\n", "# Add jittered points (strip plot)\n", "fig.add_trace(\n", " px.strip(\n", - " filtered_data_FL,\n", - " x='empo_3',\n", - " y='adiv_observed_otus',\n", - " stripmode='overlay'\n", + " filtered_data_FL, x=\"empo_3\", y=\"adiv_observed_otus\", stripmode=\"overlay\"\n", " ).data[0]\n", ")\n", "\n", "# Modify the dot color and size directly inside the add_trace()\n", - "fig.data[-1].update(\n", - " marker=dict(\n", - " color='black', \n", - " size=1, \n", - " opacity=0.7 \n", - " )\n", - ")\n", + "fig.data[-1].update(marker=dict(color=\"black\", size=1, opacity=0.7))\n", "\n", "# Add median line\n", - "median = filtered_data_FL['adiv_observed_otus'].median()\n", - "fig.add_hline(y=median, line=dict(color='yellow'))\n", + "median = filtered_data_FL[\"adiv_observed_otus\"].median()\n", + "fig.add_hline(y=median, line=dict(color=\"yellow\"))\n", "\n", "# Customize the plot\n", "fig.update_layout(\n", - " xaxis_title='',\n", - " yaxis_title='Observed tag sequences',\n", + " xaxis_title=\"\",\n", + " yaxis_title=\"Observed tag sequences\",\n", " xaxis_tickangle=-45,\n", - " plot_bgcolor='rgba(0,0,0,0)',\n", + " plot_bgcolor=\"rgba(0,0,0,0)\",\n", " showlegend=False,\n", " height=600,\n", " font=dict(size=14),\n", ")\n", "\n", "# Save figure as JSON\n", - "alpha_div_box_plot_free_living_json = os.path.join(alpha_div_output_dir, \"2_alpha_diversity_free_living_samples.json\")\n", + "alpha_div_box_plot_free_living_json = os.path.join(\n", + " alpha_div_output_dir, \"2_alpha_diversity_free_living_samples.json\"\n", + ")\n", "fig.write_json(alpha_div_box_plot_free_living_json)\n", "\n", "# Save figure as PNG\n", @@ -764,14 +812,20 @@ "outputs": [], "source": [ "# Create the output directory for the metagenomics section and average copy number subsection\n", - "avg_copy_numb_dir = os.path.join(base_output_dir, \"2_Metagenomics/2_average_copy_number\")\n", + "avg_copy_numb_dir = os.path.join(\n", + " base_output_dir, \"2_Metagenomics/2_average_copy_number\"\n", + ")\n", "os.makedirs(avg_copy_numb_dir, exist_ok=True)\n", "\n", - "# Load data \n", + "# Load data\n", "emp_gg_otus_sampsum = \"https://raw.githubusercontent.com//biocore/emp/master/data/predicted-rrna-copy-number/emp_cr_gg_13_8.qc_filtered_filt_summary_samplesum.txt\"\n", - "emp_gg_otus_sampsum_df = pd.read_csv(emp_gg_otus_sampsum, sep='\\t', index_col=0, header=None).sort_index()\n", + "emp_gg_otus_sampsum_df = pd.read_csv(\n", + " emp_gg_otus_sampsum, sep=\"\\t\", index_col=0, header=None\n", + ").sort_index()\n", "emp_gg_otus_norm_sampsum = \"https://raw.githubusercontent.com//biocore/emp/master/data/predicted-rrna-copy-number/emp_cr_gg_13_8.normalized_qcfilt_summary_samplesum.txt\"\n", - "emp_gg_otus_norm_sampsum_df = pd.read_csv(emp_gg_otus_norm_sampsum, sep='\\t', index_col=0, header=None).sort_index()" + "emp_gg_otus_norm_sampsum_df = pd.read_csv(\n", + " emp_gg_otus_norm_sampsum, sep=\"\\t\", index_col=0, header=None\n", + ").sort_index()" ] }, { @@ -784,10 +838,14 @@ "mapping_qc_filt_merged_df = mapping_qc_filt_df.copy()\n", "\n", "# Merge new mapping df with emp_gg_otus_sampsum and emp_gg_otus_norm_sampsum\n", - "mapping_qc_filt_merged_df['sampsum'] = emp_gg_otus_sampsum_df[1]\n", - "mapping_qc_filt_merged_df['normsampsum'] = emp_gg_otus_norm_sampsum_df[1]\n", - "mapping_qc_filt_merged_df['copynumberdepletion'] = np.divide(emp_gg_otus_norm_sampsum_df[1], emp_gg_otus_sampsum_df[1])\n", - "mapping_qc_filt_merged_df['averagecopy'] = np.divide(1,np.divide(emp_gg_otus_norm_sampsum_df[1],emp_gg_otus_sampsum_df[1]))" + "mapping_qc_filt_merged_df[\"sampsum\"] = emp_gg_otus_sampsum_df[1]\n", + "mapping_qc_filt_merged_df[\"normsampsum\"] = emp_gg_otus_norm_sampsum_df[1]\n", + "mapping_qc_filt_merged_df[\"copynumberdepletion\"] = np.divide(\n", + " emp_gg_otus_norm_sampsum_df[1], emp_gg_otus_sampsum_df[1]\n", + ")\n", + "mapping_qc_filt_merged_df[\"averagecopy\"] = np.divide(\n", + " 1, np.divide(emp_gg_otus_norm_sampsum_df[1], emp_gg_otus_sampsum_df[1])\n", + ")" ] }, { @@ -805,25 +863,35 @@ "source": [ "plt.figure(figsize=(10, 6))\n", "\n", - "for i in ['Animal', 'Non-saline', 'Plant', 'Saline']:\n", - " plt.hist(mapping_qc_filt_merged_df[mapping_qc_filt_merged_df.empo_2 == i]['averagecopy'].dropna(), label=i,\n", - " bins=200, linewidth=0, color=get_empo_cat_color(i), alpha=0.8)\n", + "for i in [\"Animal\", \"Non-saline\", \"Plant\", \"Saline\"]:\n", + " plt.hist(\n", + " mapping_qc_filt_merged_df[mapping_qc_filt_merged_df.empo_2 == i][\n", + " \"averagecopy\"\n", + " ].dropna(),\n", + " label=i,\n", + " bins=200,\n", + " linewidth=0,\n", + " color=get_empo_cat_color(i),\n", + " alpha=0.8,\n", + " )\n", "\n", "# Customize axes: remove top and right borders\n", - "plt.gca().spines['top'].set_visible(False)\n", - "plt.gca().spines['right'].set_visible(False)\n", + "plt.gca().spines[\"top\"].set_visible(False)\n", + "plt.gca().spines[\"right\"].set_visible(False)\n", "\n", "# Titles and labels\n", - "plt.legend(loc=1, prop={'size':9}, frameon=False)\n", - "plt.xlabel('Predicted average community 16S copy number', fontsize=12)\n", - "plt.ylabel('Number of samples', fontsize=12)\n", + "plt.legend(loc=1, prop={\"size\": 9}, frameon=False)\n", + "plt.xlabel(\"Predicted average community 16S copy number\", fontsize=12)\n", + "plt.ylabel(\"Number of samples\", fontsize=12)\n", "plt.xticks(fontsize=10)\n", "plt.yticks(fontsize=10)\n", - "plt.xlim([0,8])\n", + "plt.xlim([0, 8])\n", "plt.tight_layout()\n", "\n", "# Save the figure\n", - "avg_copy_numb_empo2 = os.path.join(avg_copy_numb_dir, \"1_average_copy_number_emp_ontology_level2.png\")\n", + "avg_copy_numb_empo2 = os.path.join(\n", + " avg_copy_numb_dir, \"1_average_copy_number_emp_ontology_level2.png\"\n", + ")\n", "plt.savefig(avg_copy_numb_empo2, dpi=300, bbox_inches=\"tight\")" ] }, @@ -847,11 +915,13 @@ "for i in mapping_qc_filt_merged_df.empo_3.dropna().unique():\n", " hist_traces.append(\n", " go.Histogram(\n", - " x=mapping_qc_filt_merged_df[mapping_qc_filt_merged_df.empo_3 == i]['averagecopy'].dropna(),\n", + " x=mapping_qc_filt_merged_df[mapping_qc_filt_merged_df.empo_3 == i][\n", + " \"averagecopy\"\n", + " ].dropna(),\n", " name=i, # Legend name\n", " marker=dict(color=get_empo_cat_color(i)), # Assign color\n", " opacity=0.5,\n", - " nbinsx=200 # Number of bins\n", + " nbinsx=200, # Number of bins\n", " )\n", " )\n", "\n", @@ -863,31 +933,33 @@ " xaxis_title=\"Predicted average community 16S copy number\",\n", " yaxis_title=\"Number of samples\",\n", " xaxis=dict(\n", - " range=[0, 8], \n", + " range=[0, 8],\n", " tickfont=dict(size=10),\n", - " showline=True, \n", - " linewidth=1, \n", + " showline=True,\n", + " linewidth=1,\n", " linecolor=\"black\",\n", - " mirror=False, \n", - " showgrid=False, \n", - " zeroline=False \n", + " mirror=False,\n", + " showgrid=False,\n", + " zeroline=False,\n", " ),\n", " yaxis=dict(\n", " tickfont=dict(size=10),\n", - " showline=True, \n", - " linewidth=1, \n", + " showline=True,\n", + " linewidth=1,\n", " linecolor=\"black\",\n", - " mirror=False, \n", - " showgrid=False, \n", - " zeroline=False \n", + " mirror=False,\n", + " showgrid=False,\n", + " zeroline=False,\n", " ),\n", - " barmode=\"overlay\", \n", + " barmode=\"overlay\",\n", " showlegend=True,\n", - " legend=dict(font=dict(size=11), borderwidth=0), \n", - " plot_bgcolor=\"white\" \n", + " legend=dict(font=dict(size=11), borderwidth=0),\n", + " plot_bgcolor=\"white\",\n", ")\n", "# Save the figure as JSON\n", - "avg_copy_numb_empo3_json = os.path.join(avg_copy_numb_dir, \"2_average_copy_number_emp_ontology_level3.json\")\n", + "avg_copy_numb_empo3_json = os.path.join(\n", + " avg_copy_numb_dir, \"2_average_copy_number_emp_ontology_level3.json\"\n", + ")\n", "fig.write_json(avg_copy_numb_empo3_json)\n", "\n", "# Save the figure as PNG\n", @@ -930,8 +1002,7 @@ "nest_phylum_plantsamples = \"https://raw.githubusercontent.com//biocore/emp/master/data/nestedness/nest_phylum_Plant.csv\"\n", "nest_phylum_plantsamples_df = pd.read_csv(nest_phylum_plantsamples)\n", "nest_phylum_nonsalinesamples = \"https://raw.githubusercontent.com//biocore/emp/master/data/nestedness/nest_phylum_Non-saline.csv\"\n", - "nest_phylum_nonsalinesamples_df = pd.read_csv(nest_phylum_nonsalinesamples)\n", - " " + "nest_phylum_nonsalinesamples_df = pd.read_csv(nest_phylum_nonsalinesamples)" ] }, { @@ -948,10 +1019,14 @@ "outputs": [], "source": [ "# Obtain a randome sample of the nestedness df for all samples\n", - "sample_nest_phylum_allsamples_df = nest_phylum_allsamples_df.sample(100, random_state=42)\n", + "sample_nest_phylum_allsamples_df = nest_phylum_allsamples_df.sample(\n", + " 100, random_state=42\n", + ")\n", "\n", "# Export the sample df as a CSV file\n", - "sample_nest_phylum_allsamples_df.to_csv(f'{nestedness_dir}/1_nestedness_random_subset.csv')" + "sample_nest_phylum_allsamples_df.to_csv(\n", + " f\"{nestedness_dir}/1_nestedness_random_subset.csv\"\n", + ")" ] }, { @@ -972,7 +1047,9 @@ "ymax = nest_phylum_allsamples_df.OBSERVATION_RANK.max()\n", "\n", "# Get colors for each empo_3 category\n", - "nest_phylum_allsamples_df['color'] = nest_phylum_allsamples_df['empo_3'].apply(get_empo_cat_color)\n", + "nest_phylum_allsamples_df[\"color\"] = nest_phylum_allsamples_df[\"empo_3\"].apply(\n", + " get_empo_cat_color\n", + ")\n", "\n", "# Create the scatter plot\n", "fig = px.scatter(\n", @@ -980,9 +1057,15 @@ " x=\"SAMPLE_RANK\",\n", " y=\"OBSERVATION_RANK\",\n", " color=\"empo_3\",\n", - " color_discrete_map={empo: get_empo_cat_color(empo) for empo in nest_phylum_allsamples_df['empo_3'].unique()},\n", - " labels={\"SAMPLE_RANK\": \"All samples (sorted by richness)\", \"OBSERVATION_RANK\": \"Phyla (sorted by prevalence)\"},\n", - " template=\"plotly_white\"\n", + " color_discrete_map={\n", + " empo: get_empo_cat_color(empo)\n", + " for empo in nest_phylum_allsamples_df[\"empo_3\"].unique()\n", + " },\n", + " labels={\n", + " \"SAMPLE_RANK\": \"All samples (sorted by richness)\",\n", + " \"OBSERVATION_RANK\": \"Phyla (sorted by prevalence)\",\n", + " },\n", + " template=\"plotly_white\",\n", ")\n", "\n", "# Customize layout\n", @@ -990,9 +1073,9 @@ "fig.update_layout(\n", " width=1200, # Increase width for a wider plot\n", " height=600, # Adjust height if needed\n", - " xaxis=dict(range=[0, xmax+1], title_font=dict(size=20), tickfont=dict(size=18)),\n", - " yaxis=dict(range=[0, ymax+0.8], title_font=dict(size=20), tickfont=dict(size=18)),\n", - " showlegend=False\n", + " xaxis=dict(range=[0, xmax + 1], title_font=dict(size=20), tickfont=dict(size=18)),\n", + " yaxis=dict(range=[0, ymax + 0.8], title_font=dict(size=20), tickfont=dict(size=18)),\n", + " showlegend=False,\n", ")\n", "\n", "# Save the figure as an interactive HTML file\n", @@ -1015,7 +1098,9 @@ "ymax = nest_phylum_plantsamples_df.OBSERVATION_RANK.max()\n", "\n", "# Get colors for each empo_3 category\n", - "nest_phylum_plantsamples_df['color'] = nest_phylum_plantsamples_df['empo_3'].apply(get_empo_cat_color)\n", + "nest_phylum_plantsamples_df[\"color\"] = nest_phylum_plantsamples_df[\"empo_3\"].apply(\n", + " get_empo_cat_color\n", + ")\n", "\n", "# Create the scatter plot\n", "fig = px.scatter(\n", @@ -1023,9 +1108,15 @@ " x=\"SAMPLE_RANK\",\n", " y=\"OBSERVATION_RANK\",\n", " color=\"empo_3\",\n", - " color_discrete_map={empo: get_empo_cat_color(empo) for empo in nest_phylum_plantsamples_df['empo_3'].unique()},\n", - " labels={\"SAMPLE_RANK\": \"Plant samples (sorted by richness)\", \"OBSERVATION_RANK\": \"Phyla (sorted by prevalence)\"},\n", - " template=\"plotly_white\"\n", + " color_discrete_map={\n", + " empo: get_empo_cat_color(empo)\n", + " for empo in nest_phylum_plantsamples_df[\"empo_3\"].unique()\n", + " },\n", + " labels={\n", + " \"SAMPLE_RANK\": \"Plant samples (sorted by richness)\",\n", + " \"OBSERVATION_RANK\": \"Phyla (sorted by prevalence)\",\n", + " },\n", + " template=\"plotly_white\",\n", ")\n", "\n", "# Customize layout\n", @@ -1033,15 +1124,17 @@ "fig.update_layout(\n", " width=1200, # Increase width for a wider plot\n", " height=600, # Adjust height if needed\n", - " xaxis=dict(range=[0, xmax+1], title_font=dict(size=20), tickfont=dict(size=18)),\n", - " yaxis=dict(range=[0, ymax+0.8], title_font=dict(size=20), tickfont=dict(size=18)),\n", + " xaxis=dict(range=[0, xmax + 1], title_font=dict(size=20), tickfont=dict(size=18)),\n", + " yaxis=dict(range=[0, ymax + 0.8], title_font=dict(size=20), tickfont=dict(size=18)),\n", " legend=dict(\n", " orientation=\"h\", # Horizontal legend\n", - " yanchor=\"top\", y=-0.2, # Moves the legend below the x-axis\n", - " xanchor=\"center\", x=0.5, # Centers the legend\n", - " font=dict(size=16)\n", + " yanchor=\"top\",\n", + " y=-0.2, # Moves the legend below the x-axis\n", + " xanchor=\"center\",\n", + " x=0.5, # Centers the legend\n", + " font=dict(size=16),\n", " ),\n", - " legend_title_text=\"\"\n", + " legend_title_text=\"\",\n", ")\n", "\n", "# Save the figure as an interactive HTML file\n", @@ -1071,7 +1164,7 @@ "ymax = nest_phylum_animalsamples_df.OBSERVATION_RANK.max()\n", "\n", "# Create the figure and axis\n", - "fig, ax = plt.subplots(figsize=(500/30, 80/12.7)) # Adjust size as needed\n", + "fig, ax = plt.subplots(figsize=(500 / 30, 80 / 12.7)) # Adjust size as needed\n", "\n", "# Store legend handles and labels\n", "legend_handles = []\n", @@ -1081,9 +1174,16 @@ "for empo3 in np.sort(nest_phylum_animalsamples_df.empo_3.unique()):\n", " color = get_empo_cat_color(empo3)\n", " scatter = ax.scatter(\n", - " nest_phylum_animalsamples_df[nest_phylum_animalsamples_df.empo_3 == empo3].SAMPLE_RANK, \n", - " nest_phylum_animalsamples_df[nest_phylum_animalsamples_df.empo_3 == empo3].OBSERVATION_RANK, \n", - " marker='|', linewidths=2, label=empo3, color=color\n", + " nest_phylum_animalsamples_df[\n", + " nest_phylum_animalsamples_df.empo_3 == empo3\n", + " ].SAMPLE_RANK,\n", + " nest_phylum_animalsamples_df[\n", + " nest_phylum_animalsamples_df.empo_3 == empo3\n", + " ].OBSERVATION_RANK,\n", + " marker=\"|\",\n", + " linewidths=2,\n", + " label=empo3,\n", + " color=color,\n", " )\n", " legend_handles.append(scatter)\n", " legend_labels.append(empo3)\n", @@ -1091,20 +1191,26 @@ "# Customize labels and appearance\n", "ax.set_xlabel(\"Animal samples (sorted by richness)\", fontsize=20)\n", "ax.set_ylabel(\"Phyla (sorted by prevalence)\", fontsize=20)\n", - "ax.tick_params(axis='both', which='major', labelsize=18)\n", + "ax.tick_params(axis=\"both\", which=\"major\", labelsize=18)\n", "\n", "# Add legend\n", "ax.legend(\n", - " handles=legend_handles, labels=legend_labels,\n", - " loc='upper center', bbox_to_anchor=(0.5, -0.2), # Moves legend below x-axis\n", - " ncol=3, fontsize=16, frameon=False, scatterpoints=1, handletextpad=0.5\n", + " handles=legend_handles,\n", + " labels=legend_labels,\n", + " loc=\"upper center\",\n", + " bbox_to_anchor=(0.5, -0.2), # Moves legend below x-axis\n", + " ncol=3,\n", + " fontsize=16,\n", + " frameon=False,\n", + " scatterpoints=1,\n", + " handletextpad=0.5,\n", ")\n", "\n", "# Increase space at the bottom so the legend is not cut off\n", "plt.subplots_adjust(bottom=0.25)\n", "\n", - "ax.set_xlim([0, xmax+1])\n", - "ax.set_ylim([0, ymax+0.8])\n", + "ax.set_xlim([0, xmax + 1])\n", + "ax.set_ylim([0, ymax + 0.8])\n", "\n", "plt.tight_layout()\n", "fig.patch.set_alpha(0.0)\n", @@ -1126,7 +1232,7 @@ "ymax = nest_phylum_nonsalinesamples_df.OBSERVATION_RANK.max()\n", "\n", "# Create the figure and axis\n", - "fig, ax = plt.subplots(figsize=(500/30, 80/12.7)) # Adjust size as needed\n", + "fig, ax = plt.subplots(figsize=(500 / 30, 80 / 12.7)) # Adjust size as needed\n", "\n", "# Store legend handles and labels\n", "legend_handles = []\n", @@ -1136,9 +1242,16 @@ "for empo3 in np.sort(nest_phylum_nonsalinesamples_df.empo_3.unique()):\n", " color = get_empo_cat_color(empo3)\n", " scatter = ax.scatter(\n", - " nest_phylum_nonsalinesamples_df[nest_phylum_nonsalinesamples_df.empo_3 == empo3].SAMPLE_RANK, \n", - " nest_phylum_nonsalinesamples_df[nest_phylum_nonsalinesamples_df.empo_3 == empo3].OBSERVATION_RANK, \n", - " marker='|', linewidths=2, label=empo3, color=color\n", + " nest_phylum_nonsalinesamples_df[\n", + " nest_phylum_nonsalinesamples_df.empo_3 == empo3\n", + " ].SAMPLE_RANK,\n", + " nest_phylum_nonsalinesamples_df[\n", + " nest_phylum_nonsalinesamples_df.empo_3 == empo3\n", + " ].OBSERVATION_RANK,\n", + " marker=\"|\",\n", + " linewidths=2,\n", + " label=empo3,\n", + " color=color,\n", " )\n", " legend_handles.append(scatter)\n", " legend_labels.append(empo3)\n", @@ -1146,27 +1259,35 @@ "# Customize labels and appearance\n", "ax.set_xlabel(\"Non saline samples (sorted by richness)\", fontsize=20)\n", "ax.set_ylabel(\"Phyla (sorted by prevalence)\", fontsize=20)\n", - "ax.tick_params(axis='both', which='major', labelsize=18)\n", + "ax.tick_params(axis=\"both\", which=\"major\", labelsize=18)\n", "\n", "# Add legend\n", "ax.legend(\n", - " handles=legend_handles, labels=legend_labels,\n", - " loc='upper center', bbox_to_anchor=(0.5, -0.2), # Moves legend below x-axis\n", - " ncol=3, fontsize=16, frameon=False, scatterpoints=1, handletextpad=0.5\n", + " handles=legend_handles,\n", + " labels=legend_labels,\n", + " loc=\"upper center\",\n", + " bbox_to_anchor=(0.5, -0.2), # Moves legend below x-axis\n", + " ncol=3,\n", + " fontsize=16,\n", + " frameon=False,\n", + " scatterpoints=1,\n", + " handletextpad=0.5,\n", ")\n", "\n", "# Increase space at the bottom so the legend is not cut off\n", "plt.subplots_adjust(bottom=0.25)\n", "\n", - "ax.set_xlim([0, xmax+1])\n", - "ax.set_ylim([0, ymax+0.8])\n", + "ax.set_xlim([0, xmax + 1])\n", + "ax.set_ylim([0, ymax + 0.8])\n", "\n", "plt.tight_layout()\n", "fig.patch.set_alpha(0.0)\n", "\n", "# Save the figure\n", "os.makedirs(nestedness_dir, exist_ok=True)\n", - "nest_phylum_nonsalinesamples_path = os.path.join(nestedness_dir, \"5_non_saline_samples.png\")\n", + "nest_phylum_nonsalinesamples_path = os.path.join(\n", + " nestedness_dir, \"5_non_saline_samples.png\"\n", + ")\n", "plt.savefig(nest_phylum_nonsalinesamples_path, dpi=300, bbox_inches=\"tight\")" ] }, @@ -1198,7 +1319,9 @@ "outputs": [], "source": [ "# Create the output directory for the network analysis section and microbial networks subsection\n", - "network_dir = os.path.join(base_output_dir, \"3_Network_analysis/1_phyla_association_networks\")\n", + "network_dir = os.path.join(\n", + " base_output_dir, \"3_Network_analysis/1_phyla_association_networks\"\n", + ")\n", "os.makedirs(network_dir, exist_ok=True)\n", "\n", "# Load OTU counts table\n", @@ -1206,11 +1329,16 @@ "\n", "# Download the file and save it as a binary file\n", "response = requests.get(otu_counts)\n", - "with open(\"example_data/Earth_microbiome_vuegen_demo_notebook/emp_deblur_100bp.subset_2k.rare_5000.biom\", 'wb') as f:\n", + "with open(\n", + " \"example_data/Earth_microbiome_vuegen_demo_notebook/emp_deblur_100bp.subset_2k.rare_5000.biom\",\n", + " \"wb\",\n", + ") as f:\n", " f.write(response.content)\n", "\n", "# Load the BIOM file and convert it to a DataFrame\n", - "otu_counts_table = biom.load_table(\"example_data/Earth_microbiome_vuegen_demo_notebook/emp_deblur_100bp.subset_2k.rare_5000.biom\")" + "otu_counts_table = biom.load_table(\n", + " \"example_data/Earth_microbiome_vuegen_demo_notebook/emp_deblur_100bp.subset_2k.rare_5000.biom\"\n", + ")" ] }, { @@ -1221,8 +1349,8 @@ "source": [ "# Collapse the table to the phylum level\n", "phylum_idx = 1\n", - "collapse_f = lambda id_, md: '; '.join(md['taxonomy'][:phylum_idx + 1])\n", - "phyla_table = otu_counts_table.collapse(collapse_f, axis='observation')\n", + "collapse_f = lambda id_, md: \"; \".join(md[\"taxonomy\"][: phylum_idx + 1])\n", + "phyla_table = otu_counts_table.collapse(collapse_f, axis=\"observation\")\n", "\n", "# Convert the collapsed table to a DataFrame\n", "phyla_counts_df = phyla_table.to_dataframe()" @@ -1242,17 +1370,23 @@ "outputs": [], "source": [ "# Clean the index (which contains Phylum names) by removing unnecessary parts\n", - "phyla_counts_df.index = phyla_counts_df.index.str.split(';').str[-1].str.replace('p__', '', regex=False)\n", + "phyla_counts_df.index = (\n", + " phyla_counts_df.index.str.split(\";\").str[-1].str.replace(\"p__\", \"\", regex=False)\n", + ")\n", "\n", "# Remove special characters like [] and unnecessary spaces\n", - "phyla_counts_df.index = phyla_counts_df.index.str.replace('[', '', regex=False).str.replace(']', '', regex=False).str.strip()\n", + "phyla_counts_df.index = (\n", + " phyla_counts_df.index.str.replace(\"[\", \"\", regex=False)\n", + " .str.replace(\"]\", \"\", regex=False)\n", + " .str.strip()\n", + ")\n", "\n", "# Remove rows where the index only has 'k__' and 'Unclassified'\n", - "phyla_counts_df = phyla_counts_df[~(phyla_counts_df.index == 'Unclassified')]\n", - "phyla_counts_df = phyla_counts_df[~phyla_counts_df.index.str.contains('k__')]\n", + "phyla_counts_df = phyla_counts_df[~(phyla_counts_df.index == \"Unclassified\")]\n", + "phyla_counts_df = phyla_counts_df[~phyla_counts_df.index.str.contains(\"k__\")]\n", "\n", "# Remove duplicaye rows\n", - "phyla_counts_df = phyla_counts_df[~phyla_counts_df.index.duplicated(keep='first')]\n" + "phyla_counts_df = phyla_counts_df[~phyla_counts_df.index.duplicated(keep=\"first\")]" ] }, { @@ -1265,7 +1399,7 @@ "sample_phyla_counts_df = phyla_counts_df.sample(50, axis=1)\n", "\n", "# Export the sample df as a CSV file\n", - "sample_phyla_counts_df.to_csv(f'{network_dir}/1_phyla_counts_subset.csv')" + "sample_phyla_counts_df.to_csv(f\"{network_dir}/1_phyla_counts_subset.csv\")" ] }, { @@ -1314,10 +1448,15 @@ "# Remove singleton nodes (nodes with no edges)\n", "G.remove_nodes_from(list(nx.isolates(G)))\n", "\n", - "# Export network as an edge list in CSV format, the \"edge_list\" word should be in the file name to be \n", + "# Export network as an edge list in CSV format, the \"edge_list\" word should be in the file name to be\n", "# recognized as an edge list file\n", "edge_list = nx.to_pandas_edgelist(G)\n", - "edge_list.to_csv(os.path.join(network_dir, \"2_phyla_correlation_network_with_0.5_threshold_edgelist.csv\"), index=False)" + "edge_list.to_csv(\n", + " os.path.join(\n", + " network_dir, \"2_phyla_correlation_network_with_0.5_threshold_edgelist.csv\"\n", + " ),\n", + " index=False,\n", + ")" ] }, { @@ -1333,19 +1472,24 @@ "metadata": {}, "outputs": [], "source": [ - "# Draw the network \n", + "# Draw the network\n", "plt.figure(figsize=(8, 6))\n", "pos = nx.kamada_kawai_layout(G) # Layout for better visualization\n", - "nx.draw(G, pos, \n", - " with_labels=True, \n", - " node_size=500, \n", - " node_color=\"lightblue\", \n", - " edgecolors=\"black\",\n", - " linewidths=0.3, \n", - " font_size=10)\n", + "nx.draw(\n", + " G,\n", + " pos,\n", + " with_labels=True,\n", + " node_size=500,\n", + " node_color=\"lightblue\",\n", + " edgecolors=\"black\",\n", + " linewidths=0.3,\n", + " font_size=10,\n", + ")\n", "\n", "# Export the figure as a PNG file\n", - "network_path = os.path.join(network_dir, \"3_phyla_correlation_network_with_0.5_threshold.png\")\n", + "network_path = os.path.join(\n", + " network_dir, \"3_phyla_correlation_network_with_0.5_threshold.png\"\n", + ")\n", "plt.savefig(network_path, dpi=300, bbox_inches=\"tight\")" ] }, @@ -1382,7 +1526,9 @@ "source": [ "# Generate the report\n", "report_type = \"streamlit\"\n", - "_ = report_generator.get_report(dir_path = base_output_dir, report_type = report_type, logger = None)" + "_ = report_generator.get_report(\n", + " dir_path=base_output_dir, report_type=report_type, logger=None\n", + ")" ] }, { @@ -1395,14 +1541,20 @@ "# run_streamlit = True # uncomment line to run the streamlit report\n", "# Launch the Streamlit report depneding on the platform\n", "if not IN_COLAB and run_streamlit:\n", - " !streamlit run streamlit_report/sections/report_manager.py\n", + " !streamlit run streamlit_report/sections/report_manager.py\n", "elif run_streamlit:\n", - " # see: https://discuss.streamlit.io/t/how-to-launch-streamlit-app-from-google-colab-notebook/42399\n", - " print(\"Password/Enpoint IP for localtunnel is:\",urllib.request.urlopen('https://ipv4.icanhazip.com').read().decode('utf8').strip(\"\\n\"))\n", - " # Run the Streamlit app in the background\n", - " !streamlit run streamlit_report/sections/report_manager.py --server.address=localhost &>/content/logs.txt &\n", - " # Expose the Streamlit app on port 8501\n", - " !npx localtunnel --port 8501 --subdomain vuegen-demo\n", + " # see: https://discuss.streamlit.io/t/how-to-launch-streamlit-app-from-google-colab-notebook/42399\n", + " print(\n", + " \"Password/Enpoint IP for localtunnel is:\",\n", + " urllib.request.urlopen(\"https://ipv4.icanhazip.com\")\n", + " .read()\n", + " .decode(\"utf8\")\n", + " .strip(\"\\n\"),\n", + " )\n", + " # Run the Streamlit app in the background\n", + " !streamlit run streamlit_report/sections/report_manager.py --server.address=localhost &>/content/logs.txt &\n", + " # Expose the Streamlit app on port 8501\n", + " !npx localtunnel --port 8501 --subdomain vuegen-demo\n", "else:\n", " print(\"Streamlit report not executed, set run_streamlit to True to run the report\")" ] @@ -1448,7 +1600,9 @@ "metadata": {}, "outputs": [], "source": [ - "empo_logo_path = \"https://raw.githubusercontent.com/ElDeveloper/cogs220/master/emp-logo.svg\"\n", + "empo_logo_path = (\n", + " \"https://raw.githubusercontent.com/ElDeveloper/cogs220/master/emp-logo.svg\"\n", + ")\n", "\n", "# Load the YAML file\n", "config = load_yaml_config(config_path)\n", @@ -1472,15 +1626,19 @@ "source": [ "# Update the description for the EDA section\n", "for section in config[\"sections\"]:\n", - " if section[\"title\"] == \"Exploratory Data Analysis\": \n", - " section[\"description\"] = \"This section contains the exploratory data analysis of the Earth Microbiome Project (EMP) dataset.\"\n", + " if section[\"title\"] == \"Exploratory Data Analysis\":\n", + " section[\"description\"] = (\n", + " \"This section contains the exploratory data analysis of the Earth Microbiome Project (EMP) dataset.\"\n", + " )\n", "\n", "# Update the description for the alpha diversity subsection from the Metagenomics section\n", "for section in config[\"sections\"]:\n", " if section[\"title\"] == \"Metagenomics\":\n", " for subsection in section[\"subsections\"]:\n", " if subsection[\"title\"] == \"Alpha Diversity\":\n", - " subsection[\"description\"] = \"This subsection contains the alpha diversity analysis of the EMP dataset.\"\n" + " subsection[\"description\"] = (\n", + " \"This subsection contains the alpha diversity analysis of the EMP dataset.\"\n", + " )" ] }, { @@ -1499,11 +1657,11 @@ "# Define new plot with a URL as the file path\n", "chem_prop_plot = {\n", " \"title\": \"Physicochemical properties of the EMP samples\",\n", - " \"file_path\": \"https://raw.githubusercontent.com/biocore/emp/master/methods/images/figureED1_physicochemical.png\", \n", + " \"file_path\": \"https://raw.githubusercontent.com/biocore/emp/master/methods/images/figureED1_physicochemical.png\",\n", " \"description\": \"\",\n", " \"caption\": \"Pairwise scatter plots of available physicochemical metadat are shown for temperature, salinity, oxygen, and pH, and for phosphate, nitrate, and ammonium\",\n", " \"component_type\": \"plot\",\n", - " \"plot_type\": \"static\"\n", + " \"plot_type\": \"static\",\n", "}\n", "\n", "# Add the plot to the Sample Provenance subsection in the EDA section\n", @@ -1530,23 +1688,23 @@ "# Define new plot with a URL as the file path\n", "specif_seq_plot = {\n", " \"title\": \"Specificity of sequences and higher taxonomic groups for environment\",\n", - " \"file_path\": \"https://raw.githubusercontent.com/biocore/emp/master/methods/images/figure4_entropy.png\", \n", + " \"file_path\": \"https://raw.githubusercontent.com/biocore/emp/master/methods/images/figure4_entropy.png\",\n", " \"description\": \"\",\n", " \"caption\": \"a) Environment distribution in all genera and 400 randomly chosen tag sequence. b) and c) Shannon entropy within each taxonomic group.\",\n", " \"component_type\": \"plot\",\n", - " \"plot_type\": \"static\"\n", + " \"plot_type\": \"static\",\n", "}\n", "\n", "# Define the new subsection for the Shannon entropy analysis\n", "entropy_subsection = {\n", - " \"title\": \"Shanon entropy analysis\", \n", + " \"title\": \"Shanon entropy analysis\",\n", " \"description\": \"This subsection contains the Shannon entropy analysis of the EMP dataset.\",\n", - " \"components\": [specif_seq_plot] \n", + " \"components\": [specif_seq_plot],\n", "}\n", "\n", "# Add the new subsection to the Metagenomics section\n", "for section in config[\"sections\"]:\n", - " if section[\"title\"] == \"Metagenomics\": \n", + " if section[\"title\"] == \"Metagenomics\":\n", " section[\"subsections\"].append(entropy_subsection)\n", "\n", "# Save the modified YAML file\n", @@ -1571,7 +1729,9 @@ "source": [ "# Test the changes by generarating the report from the modified YAML file\n", "report_type = \"streamlit\"\n", - "_ = report_generator.get_report(config_path = config_path, report_type = report_type, logger = None)" + "_ = report_generator.get_report(\n", + " config_path=config_path, report_type=report_type, logger=None\n", + ")" ] }, { @@ -1584,14 +1744,20 @@ "# run_streamlit = True # uncomment line to run the streamlit report\n", "# Launch the Streamlit report depneding on the platform\n", "if not IN_COLAB and run_streamlit:\n", - " !streamlit run streamlit_report/sections/report_manager.py\n", + " !streamlit run streamlit_report/sections/report_manager.py\n", "elif run_streamlit:\n", - " # see: https://discuss.streamlit.io/t/how-to-launch-streamlit-app-from-google-colab-notebook/42399\n", - " print(\"Password/Enpoint IP for localtunnel is:\",urllib.request.urlopen('https://ipv4.icanhazip.com').read().decode('utf8').strip(\"\\n\"))\n", - " # Run the Streamlit app in the background\n", - " !streamlit run streamlit_report/sections/report_manager.py --server.address=localhost &>/content/logs.txt &\n", - " # Expose the Streamlit app on port 8501\n", - " !npx localtunnel --port 8501 --subdomain vuegen-demo\n", + " # see: https://discuss.streamlit.io/t/how-to-launch-streamlit-app-from-google-colab-notebook/42399\n", + " print(\n", + " \"Password/Enpoint IP for localtunnel is:\",\n", + " urllib.request.urlopen(\"https://ipv4.icanhazip.com\")\n", + " .read()\n", + " .decode(\"utf8\")\n", + " .strip(\"\\n\"),\n", + " )\n", + " # Run the Streamlit app in the background\n", + " !streamlit run streamlit_report/sections/report_manager.py --server.address=localhost &>/content/logs.txt &\n", + " # Expose the Streamlit app on port 8501\n", + " !npx localtunnel --port 8501 --subdomain vuegen-demo\n", "else:\n", " print(\"Streamlit report not executed, set run_streamlit to True to run the report\")" ] @@ -1611,13 +1777,15 @@ "source": [ "# Test the changes by generarating the report from the modified YAML file\n", "report_type = \"html\"\n", - "_ = report_generator.get_report(config_path = config_path, report_type = report_type, logger = None)" + "_ = report_generator.get_report(\n", + " config_path=config_path, report_type=report_type, logger=None\n", + ")" ] } ], "metadata": { "kernelspec": { - "display_name": "vuegen", + "display_name": "vuegen_py312", "language": "python", "name": "python3" }, @@ -1631,7 +1799,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.21" + "version": "3.12.9" } }, "nbformat": 4, From e5b598577254e9f140be312acc0d48eee7a5dff4 Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Tue, 18 Mar 2025 16:02:29 +0100 Subject: [PATCH 88/91] :memo: link README of GUI --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index a786313..0a01948 100644 --- a/README.md +++ b/README.md @@ -137,6 +137,9 @@ Streamlit works out of the box as a purely Python based package. For `html` crea have a global Python installation with the `jupyter` package installed. `quarto` needs to start a kernel for execution. This is also true if you install `quarto` globally on your machine. +More information can be found in the +[GUI README](https://github.com/Multiomics-Analytics-Group/vuegen/blob/os_installers/gui/README.md). + ## Case studies VueGen’s functionality is demonstrated through two case studies: From 4a191d88421da38fcf896b89b5046a3063cf5e3a Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Wed, 19 Mar 2025 08:26:13 +0100 Subject: [PATCH 89/91] =?UTF-8?q?=F0=9F=9A=A7=20gh=20release=20upload=20fo?= =?UTF-8?q?r=20adding=20executables=20to=20release=3F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/cdci.yml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cdci.yml b/.github/workflows/cdci.yml index 8bfa271..c053d33 100644 --- a/.github/workflows/cdci.yml +++ b/.github/workflows/cdci.yml @@ -134,8 +134,6 @@ jobs: python-version: ["3.12"] os: # https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/running-variations-of-jobs-in-a-workflow#example-using-a-multi-dimension-matrix - - runner: "ubuntu-latest" - label: "ubuntu-latest_LTS-x64" - runner: "macos-13" label: "macos-13-x64" - runner: "macos-15" @@ -169,3 +167,13 @@ jobs: with: name: vuegen_gui_${{ matrix.os.label }} path: gui/dist/ + - name: Upload Executable to a GitHub Release + if: startsWith(github.ref, 'refs/tags') + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + TAG_NAME=${GITHUB_REF#refs/tags/} + cp + gh release upload "$TAG_NAME" gui/dist/vuegen_gui.*#vuegen_gui_${{ matrix.os.label }} + # https://cli.github.com/manual/gh_release_upload + # either .app or .exe depending on the OS From fd4685786d5d172d5f18423394ec92211178942e Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Wed, 19 Mar 2025 11:34:08 +0100 Subject: [PATCH 90/91] :art: no time zone in log file name, use onefile for windows - time zone names on windows are too long - onefile on Windows should lead to only one exe - see if option leads to okay option for .app on MAC --- .github/workflows/cdci.yml | 2 +- src/vuegen/utils.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cdci.yml b/.github/workflows/cdci.yml index c053d33..faf8008 100644 --- a/.github/workflows/cdci.yml +++ b/.github/workflows/cdci.yml @@ -155,7 +155,7 @@ jobs: - name: Build executable run: | cd gui - pyinstaller -n vuegen_gui --windowed --collect-all pyvis --collect-all streamlit --collect-all st_aggrid --collect-all customtkinter --collect-all quarto_cli --collect-all plotly --collect-all _plotly_utils --collect-all traitlets --collect-all referencing --collect-all rpds --collect-all tenacity --collect-all pyvis --collect-all pandas --collect-all numpy --collect-all matplotlib --collect-all openpyxl --collect-all xlrd --collect-all nbformat --collect-all nbclient --collect-all altair --collect-all itables --collect-all kaleido --collect-all pyarrow --collect-all dataframe_image --collect-all narwhals --collect-all PIL --collect-all vl_convert --add-data ../docs/example_data/Basic_example_vuegen_demo_notebook:example_data/Basic_example_vuegen_demo_notebook --add-data ../docs/images/vuegen_logo.png:. app.py + pyinstaller -n vuegen_gui --onefile --windowed --collect-all pyvis --collect-all streamlit --collect-all st_aggrid --collect-all customtkinter --collect-all quarto_cli --collect-all plotly --collect-all _plotly_utils --collect-all traitlets --collect-all referencing --collect-all rpds --collect-all tenacity --collect-all pyvis --collect-all pandas --collect-all numpy --collect-all matplotlib --collect-all openpyxl --collect-all xlrd --collect-all nbformat --collect-all nbclient --collect-all altair --collect-all itables --collect-all kaleido --collect-all pyarrow --collect-all dataframe_image --collect-all narwhals --collect-all PIL --collect-all vl_convert --add-data ../docs/example_data/Basic_example_vuegen_demo_notebook:example_data/Basic_example_vuegen_demo_notebook --add-data ../docs/images/vuegen_logo.png:. app.py # --windowed only for mac, see: # https://pyinstaller.org/en/stable/usage.html#building-macos-app-bundles # 'Under macOS, PyInstaller always builds a UNIX executable in dist.' diff --git a/src/vuegen/utils.py b/src/vuegen/utils.py index d792b95..8f4e2b2 100644 --- a/src/vuegen/utils.py +++ b/src/vuegen/utils.py @@ -642,7 +642,7 @@ def generate_log_filename(folder: str = "logs", suffix: str = "") -> str: except OSError as e: raise OSError(f"Error creating directory '{folder}': {e}") # MAIN FUNCTION - log_filename = get_time(incl_timezone=True) + "_" + suffix + ".log" + log_filename = get_time(incl_timezone=False) + "_" + suffix + ".log" log_filepath = os.path.join(folder, log_filename) return log_filepath From 907f083a90285b2a73278f5cd789fed61d361940 Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Thu, 20 Mar 2025 10:54:23 +0100 Subject: [PATCH 91/91] :art: remove some comments and fix docstring --- src/vuegen/utils.py | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/src/vuegen/utils.py b/src/vuegen/utils.py index 8f4e2b2..1af328e 100644 --- a/src/vuegen/utils.py +++ b/src/vuegen/utils.py @@ -594,9 +594,7 @@ def get_time(incl_time: bool = True, incl_timezone: bool = True) -> str: the_time = datetime.now() timezone = datetime.now().astimezone().tzname() # convert date parts to string - y = str(the_time.year) - M = str(the_time.month) - d = str(the_time.day) + # putting date parts into one string if incl_time and incl_timezone: fname = the_time.isoformat(sep="_", timespec="seconds") + "_" + timezone @@ -605,19 +603,13 @@ def get_time(incl_time: bool = True, incl_timezone: bool = True) -> str: elif incl_timezone: fname = "_".join([the_time.isoformat(sep="_", timespec="hours")[:-3], timezone]) else: - fname = y + M + d + y = str(the_time.year) + m = str(the_time.month) + d = str(the_time.day) + fname = y + m + d # optional fname = fname.replace(":", "-") # remove ':' from hours, minutes, seconds - # POSTCONDITIONALS - # ! to delete (was it jused for something?) - # parts = fname.split("_") - # if incl_time and incl_timezone: - # assert len(parts) == 3, f"time and/or timezone inclusion issue: {fname}" - # elif incl_time or incl_timezone: - # assert len(parts) == 2, f"time/timezone inclusion issue: {fname}" - # else: - # assert len(parts) == 1, f"time/timezone inclusion issue: {fname}" return fname @@ -724,7 +716,7 @@ def get_logger( Returns ------- - tuple[logging.Logger, str + tuple[logging.Logger, str] A tuple containing the logger instance and the log file path. """ # Generate log file name