From e6ee50f30043bb7fb423bc1cd2800d32aabf2afd Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Tue, 23 Aug 2022 13:59:22 -0400 Subject: [PATCH 01/97] add jupyter-voila to models --- rsconnect/models.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/rsconnect/models.py b/rsconnect/models.py index 872eb6c0..6e29182d 100644 --- a/rsconnect/models.py +++ b/rsconnect/models.py @@ -78,6 +78,7 @@ class AppModes(object): SHINY_QUARTO = AppMode(13, "quarto-shiny", "Shiny Quarto Document") STATIC_QUARTO = AppMode(14, "quarto-static", "Quarto Document", ".qmd") PYTHON_SHINY = AppMode(15, "python-shiny", "Python Shiny Application") + JUPYTER_VOILA = AppMode(16, "jupyter-voila", "Python Shiny Application") _modes = [ UNKNOWN, @@ -96,6 +97,7 @@ class AppModes(object): SHINY_QUARTO, STATIC_QUARTO, PYTHON_SHINY, + JUPYTER_VOILA, ] _cloud_to_connect_modes = { From d5cab0c1cf5e144f14fb52c757e521b75aa4b485 Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Tue, 23 Aug 2022 15:17:05 -0400 Subject: [PATCH 02/97] wrtie-manifest for voila --- rsconnect/bundle.py | 46 +++++++++++++++++++++++ rsconnect/main.py | 89 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 135 insertions(+) diff --git a/rsconnect/bundle.py b/rsconnect/bundle.py index afb8aa64..5ab50372 100644 --- a/rsconnect/bundle.py +++ b/rsconnect/bundle.py @@ -1266,6 +1266,52 @@ def write_notebook_manifest_json( return exists(join(directory, environment.filename)) +def write_voila_manifest_json( + entry_point_file: str, + environment: Environment, + app_mode: AppMode, + extra_files: typing.List[str], + image: str = None, +) -> bool: + """ + Creates and writes a manifest.json file for the given entry point file. If + the application mode is not provided, an attempt will be made to resolve one + based on the extension portion of the entry point file. + + :param entry_point_file: the entry point file (Voila notebook, etc.) to build + the manifest for. + :param environment: the Python environment to start with. This should be what's + returned by the inspect_environment() function. + :param app_mode: the application mode to assume. If this is None, the extension + portion of the entry point file name will be used to derive one. Previous default = None. + :param extra_files: any extra files that should be included in the manifest. Previous default = None. + :param image: the optional docker image to be specified for off-host execution. Default = None. + :return: whether or not the environment file (requirements.txt, environment.yml, + etc.) that goes along with the manifest exists. + """ + extra_files = validate_extra_files(dirname(entry_point_file), extra_files) + directory = dirname(entry_point_file) + file_name = basename(entry_point_file) + manifest_path = join(directory, "manifest.json") + + if app_mode is None: + _, extension = splitext(file_name) + app_mode = AppModes.get_by_extension(extension, True) + if app_mode == AppModes.UNKNOWN: + raise RSConnectException('Could not determine the app mode from "%s"; please specify one.' % extension) + + manifest_data = make_source_manifest(app_mode, environment, file_name, None, image) + manifest_add_file(manifest_data, file_name, directory) + manifest_add_buffer(manifest_data, environment.filename, environment.contents) + + for rel_path in extra_files: + manifest_add_file(manifest_data, rel_path, directory) + + write_manifest_json(manifest_path, manifest_data) + + return exists(join(directory, environment.filename)) + + def create_api_manifest_and_environment_file( directory: str, entry_point: str, diff --git a/rsconnect/main.py b/rsconnect/main.py index 97b431d9..822eed47 100644 --- a/rsconnect/main.py +++ b/rsconnect/main.py @@ -52,6 +52,7 @@ write_api_manifest_json, write_environment_file, write_quarto_manifest_json, + write_voila_manifest_json, validate_entry_point, validate_extra_files, validate_file_is_notebook, @@ -1247,6 +1248,94 @@ def write_manifest_notebook( write_environment_file(environment, base_dir) +@write_manifest.command( + name="voila", + short_help="Create a manifest.json file for a Voila notebook.", + help=( + "Create a manifest.json file for a Voila notebook for later deployment. " + 'This will create an environment file ("requirements.txt") if one does ' + "not exist. All files are created in the same directory as the notebook file." + ), +) +@click.option("--overwrite", "-o", is_flag=True, help="Overwrite manifest.json, if it exists.") +@click.option( + "--python", + "-p", + type=click.Path(exists=True), + help="Path to Python interpreter whose environment should be used. " + + "The Python environment must have the rsconnect package installed.", +) +@click.option( + "--conda", + "-C", + is_flag=True, + hidden=True, + help="Use Conda to deploy (requires RStudio Connect version 1.8.2 or later)", +) +@click.option( + "--force-generate", + "-g", + is_flag=True, + help='Force generating "requirements.txt", even if it already exists.', +) +@click.option("--verbose", "-v", "verbose", is_flag=True, help="Print detailed messages") +@click.option( + "--image", + "-I", + help="Target image to be used during content execution (only applicable if the RStudio Connect " + "server is configured to use off-host execution)", +) +@click.argument("file", type=click.Path(exists=True, dir_okay=False, file_okay=True)) +@click.argument( + "extra_files", + nargs=-1, + type=click.Path(exists=True, dir_okay=False, file_okay=True), +) +def write_manifest_voila( + overwrite, + python, + conda, + force_generate, + verbose, + file, + extra_files, + image, +): + set_verbosity(verbose) + with cli_feedback("Checking arguments"): + validate_file_is_notebook(file) + + base_dir = dirname(file) + extra_files = validate_extra_files(base_dir, extra_files) + manifest_path = join(base_dir, "manifest.json") + + if exists(manifest_path) and not overwrite: + raise RSConnectException("manifest.json already exists. Use --overwrite to overwrite.") + + with cli_feedback("Inspecting Python environment"): + python, environment = get_python_env_info(file, python, conda, force_generate) + + _warn_on_ignored_conda_env(environment) + + with cli_feedback("Creating manifest.json"): + environment_file_exists = write_voila_manifest_json( + file, + environment, + AppModes.JUPYTER_VOILA, + extra_files, + image, + ) + + if environment_file_exists and not force_generate: + click.secho( + " Warning: %s already exists and will not be overwritten." % environment.filename, + fg="yellow", + ) + else: + with cli_feedback("Creating %s" % environment.filename): + write_environment_file(environment, base_dir) + + @write_manifest.command( name="quarto", short_help="Create a manifest.json file for Quarto content.", From ac20ec0bc2157ccae5305537bf7807e1db91f7d2 Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Thu, 25 Aug 2022 15:06:22 -0400 Subject: [PATCH 03/97] add deploy voila command --- rsconnect/bundle.py | 43 +++++++++++++++++++++++ rsconnect/main.py | 83 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 126 insertions(+) diff --git a/rsconnect/bundle.py b/rsconnect/bundle.py index 5ab50372..d959da72 100644 --- a/rsconnect/bundle.py +++ b/rsconnect/bundle.py @@ -317,6 +317,49 @@ def make_notebook_source_bundle( return bundle_file +def make_voila_source_bundle( + file: str, + environment: Environment, + extra_files: typing.List[str], + image: str = None, +) -> typing.IO[bytes]: + """Create a bundle containing the specified notebook and python environment. + + Returns a file-like object containing the bundle tarball. + """ + if extra_files is None: + extra_files = [] + base_dir = dirname(file) + nb_name = basename(file) + + manifest = make_source_manifest(AppModes.JUPYTER_VOILA, environment, nb_name, None, image) + manifest_add_file(manifest, nb_name, base_dir) + manifest_add_buffer(manifest, environment.filename, environment.contents) + + if extra_files: + skip = [nb_name, environment.filename, "manifest.json"] + extra_files = sorted(list(set(extra_files) - set(skip))) + + for rel_path in extra_files: + manifest_add_file(manifest, rel_path, base_dir) + + logger.debug("manifest: %r", manifest) + + bundle_file = tempfile.TemporaryFile(prefix="rsc_bundle") + with tarfile.open(mode="w:gz", fileobj=bundle_file) as bundle: + + # add the manifest first in case we want to partially untar the bundle for inspection + bundle_add_buffer(bundle, "manifest.json", json.dumps(manifest, indent=2)) + bundle_add_buffer(bundle, environment.filename, environment.contents) + bundle_add_file(bundle, nb_name, base_dir) + + for rel_path in extra_files: + bundle_add_file(bundle, rel_path, base_dir) + + bundle_file.seek(0) + return bundle_file + + def make_quarto_source_bundle( file_or_directory: str, inspect: typing.Dict[str, typing.Any], diff --git a/rsconnect/main.py b/rsconnect/main.py index 822eed47..a0cd0885 100644 --- a/rsconnect/main.py +++ b/rsconnect/main.py @@ -47,6 +47,7 @@ make_api_bundle, make_notebook_html_bundle, make_notebook_source_bundle, + make_voila_source_bundle, read_manifest_app_mode, write_notebook_manifest_json, write_api_manifest_json, @@ -734,6 +735,88 @@ def deploy_notebook( ce.deploy_bundle().save_deployed_info().emit_task_log() +# noinspection SpellCheckingInspection,DuplicatedCode +@deploy.command( + name="voila", + short_help="Deploy Voila notebook to RStudio Connect [v2022.09.0+].", + help=( + "Deploy a Voila notebook to RStudio Connect. This may be done by source or as a static HTML " + "page. If the notebook is deployed as a static HTML page (--static), it cannot be scheduled or " + "rerun on the Connect server." + ), +) +@server_args +@content_args +@click.option( + "--python", + "-p", + type=click.Path(exists=True), + help=( + "Path to Python interpreter whose environment should be used. " + "The Python environment must have the rsconnect package installed." + ), +) +@click.option( + "--force-generate", + "-g", + is_flag=True, + help='Force generating "requirements.txt", even if it already exists.', +) +@click.option( + "--image", + "-I", + help="Target image to be used during content execution (only applicable if the RStudio Connect " + "server is configured to use off-host execution)", +) +@click.argument("file", type=click.Path(exists=True, dir_okay=False, file_okay=True)) +@click.argument( + "extra_files", + nargs=-1, + type=click.Path(exists=True, dir_okay=False, file_okay=True), +) +@cli_exception_handler +def deploy_voila( + name: str, + server: str, + api_key: str, + insecure: bool, + cacert: typing.IO, + new: bool, + app_id: str, + title: str, + python, + force_generate, + verbose: bool, + file: str, + extra_files, + env_vars: typing.Dict[str, str], + image: str, +): + kwargs = locals() + set_verbosity(verbose) + + kwargs["extra_files"] = extra_files = validate_extra_files(dirname(file), extra_files) + app_mode = AppModes.JUPYTER_VOILA + + base_dir = dirname(file) + _warn_on_ignored_manifest(base_dir) + _warn_if_no_requirements_file(base_dir) + _warn_if_environment_directory(base_dir) + python, environment = get_python_env_info(file, python, conda_mode=False, force_generate=force_generate) + + if force_generate: + _warn_on_ignored_requirements(base_dir, environment.filename) + + ce = RSConnectExecutor(**kwargs).validate_server().validate_app_mode(app_mode=app_mode) + ce.make_bundle( + make_voila_source_bundle, + file, + environment, + extra_files, + image=image, + ).deploy_bundle().save_deployed_info().emit_task_log() + + # noinspection SpellCheckingInspection,DuplicatedCode @deploy.command( name="manifest", From aa703f806ae48be74c6158078deae0fc373fe034 Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Fri, 26 Aug 2022 15:54:51 -0400 Subject: [PATCH 04/97] mimetype inference no longer requires set defaults --- rsconnect/bundle.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/rsconnect/bundle.py b/rsconnect/bundle.py index d959da72..a033d8cc 100644 --- a/rsconnect/bundle.py +++ b/rsconnect/bundle.py @@ -755,9 +755,8 @@ def infer_entrypoint(path, mimetype): if not os.path.isdir(path): raise ValueError("Entrypoint is not a valid file type or directory.") - default_mimetype_entrypoints = {"text/html": "index.html"} - if mimetype not in default_mimetype_entrypoints: - raise ValueError("Not supported mimetype inference.") + default_mimetype_entrypoints = defaultdict(str) + default_mimetype_entrypoints["text/html"] = "index.html" mimetype_filelist = defaultdict(list) From 03d05bc194a66d10db7197176cd99fe814e48bda Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Fri, 26 Aug 2022 16:02:00 -0400 Subject: [PATCH 05/97] enable deploying voila by directory --- rsconnect/bundle.py | 113 ++++++++++++++++++++++++++++++++++++++++++++ rsconnect/main.py | 85 ++++++++++++++++++++------------- 2 files changed, 164 insertions(+), 34 deletions(-) diff --git a/rsconnect/bundle.py b/rsconnect/bundle.py index a033d8cc..ac934352 100644 --- a/rsconnect/bundle.py +++ b/rsconnect/bundle.py @@ -5,6 +5,7 @@ import hashlib import io import json +import mimetypes import os import subprocess import sys @@ -802,6 +803,118 @@ def make_html_bundle( return bundle_file +def pack_relevant_files( + path: str, + entrypoint: str, + extra_files: typing.List[str], + excludes: typing.List[str], +) -> typing.Tuple[typing.Dict[str, typing.Any], typing.List[str]]: + + """ + Packs all the extra files and filters out excludes. + + :param path: the file, or the directory containing the files to deploy. + :param entry_point: the main entry point for the API. + :param extra_files: a sequence of any extra files to include in the bundle. + :param excludes: a sequence of glob patterns that will exclude matched files. + :return: a list of the files involved. + """ + extra_files = list(extra_files) if extra_files else [] + + if path.startswith(os.curdir): + path = relpath(path) + if entrypoint.startswith(os.curdir): + entrypoint = relpath(entrypoint) + extra_files = [relpath(f) if isfile(f) and f.startswith(os.curdir) else f for f in extra_files] + + if is_environment_dir(path): + excludes = list(excludes or []) + ["bin/", "lib/"] + + extra_files = extra_files or [] + skip = ["manifest.json"] + extra_files = sorted(list(set(extra_files) - set(skip))) + + # Don't include these top-level files. + excludes = list(excludes) if excludes else [] + excludes.append("manifest.json") + if not isfile(path): + excludes.extend(list_environment_dirs(path)) + glob_set = create_glob_set(path, excludes) + + file_list = [] + + for rel_path in extra_files: + file_list.append(rel_path) + + if isfile(path): + file_list.append(path) + else: + for subdir, dirs, files in os.walk(path): + for file in files: + abs_path = os.path.join(subdir, file) + rel_path = os.path.relpath(abs_path, path) + + if keep_manifest_specified_file(rel_path) and ( + rel_path in extra_files or not glob_set.matches(abs_path) + ): + file_list.append(rel_path) + # Don't add extra files more than once. + if rel_path in extra_files: + extra_files.remove(rel_path) + + relevant_files = sorted(file_list) + + return relevant_files + + +def make_voila_bundle( + path: str, + entrypoint: str, + extra_files: typing.List[str], + excludes: typing.List[str], + environment: Environment, + image: str = None, +) -> typing.IO[bytes]: + """ + Create an voila bundle, given a path and a manifest. + + :param path: the file, or the directory containing the files to deploy. + :param entry_point: the main entry point for the API. + :param extra_files: a sequence of any extra files to include in the bundle. + :param excludes: a sequence of glob patterns that will exclude matched files. + :param image: the optional docker image to be specified for off-host execution. Default = None. + :return: a file-like object containing the bundle tarball. + """ + mimetypes.add_type("text/ipynb", ".ipynb") + entrypoint = entrypoint or infer_entrypoint(path=path, mimetype="text/ipynb") + base_dir = dirname(entrypoint) + nb_name = basename(entrypoint) + + manifest = make_source_manifest(AppModes.JUPYTER_VOILA, environment, nb_name, None, image) + manifest_add_file(manifest, nb_name, base_dir) + manifest_add_buffer(manifest, environment.filename, environment.contents) + + if extra_files: + skip = [nb_name, environment.filename, "manifest.json"] + extra_files = sorted(list(set(extra_files) - set(skip))) + relevant_files = pack_relevant_files(path, entrypoint, extra_files, excludes) + for rel_path in relevant_files: + manifest_add_file(manifest, rel_path, path) + + bundle_file = tempfile.TemporaryFile(prefix="rsc_bundle") + + with tarfile.open(mode="w:gz", fileobj=bundle_file) as bundle: + bundle_add_buffer(bundle, "manifest.json", json.dumps(manifest, indent=2)) + + for rel_path in relevant_files: + bundle_add_file(bundle, rel_path, path) + + # rewind file pointer + bundle_file.seek(0) + + return bundle_file + + def make_api_bundle( directory: str, entry_point: str, diff --git a/rsconnect/main.py b/rsconnect/main.py index a0cd0885..dfbc7e94 100644 --- a/rsconnect/main.py +++ b/rsconnect/main.py @@ -7,7 +7,7 @@ import textwrap import click from six import text_type -from os.path import abspath, dirname, exists, isdir, join +from os.path import abspath, dirname, exists, isdir, join, isfile from functools import wraps from .environment import EnvironmentException from .exception import RSConnectException @@ -47,6 +47,7 @@ make_api_bundle, make_notebook_html_bundle, make_notebook_source_bundle, + make_voila_bundle, make_voila_source_bundle, read_manifest_app_mode, write_notebook_manifest_json, @@ -768,7 +769,7 @@ def deploy_notebook( help="Target image to be used during content execution (only applicable if the RStudio Connect " "server is configured to use off-host execution)", ) -@click.argument("file", type=click.Path(exists=True, dir_okay=False, file_okay=True)) +@click.argument("path", type=click.Path(exists=True, dir_okay=True, file_okay=True)) @click.argument( "extra_files", nargs=-1, @@ -776,45 +777,61 @@ def deploy_notebook( ) @cli_exception_handler def deploy_voila( - name: str, - server: str, - api_key: str, - insecure: bool, - cacert: typing.IO, - new: bool, - app_id: str, - title: str, - python, - force_generate, - verbose: bool, - file: str, - extra_files, - env_vars: typing.Dict[str, str], - image: str, + connect_server: api.RSConnectServer = None, + path: str = None, + entrypoint: str = None, + python=None, + force_generate=False, + extra_files=None, + excludes=None, + image: str = "", + title: str = None, + env_vars: typing.Dict[str, str] = None, + verbose: bool = False, + new: bool = False, + app_id: str = None, + name: str = None, + server: str = None, + api_key: str = None, + insecure: bool = False, + cacert: typing.IO = None, ): kwargs = locals() set_verbosity(verbose) - - kwargs["extra_files"] = extra_files = validate_extra_files(dirname(file), extra_files) app_mode = AppModes.JUPYTER_VOILA + python, environment = get_python_env_info(path, python, conda_mode=False, force_generate=force_generate) + if isfile(path): + kwargs["extra_files"] = extra_files = validate_extra_files(dirname(path), extra_files) + base_dir = dirname(path) + _warn_on_ignored_manifest(base_dir) + _warn_if_no_requirements_file(base_dir) + _warn_if_environment_directory(base_dir) - base_dir = dirname(file) - _warn_on_ignored_manifest(base_dir) - _warn_if_no_requirements_file(base_dir) - _warn_if_environment_directory(base_dir) - python, environment = get_python_env_info(file, python, conda_mode=False, force_generate=force_generate) + if force_generate: + _warn_on_ignored_requirements(base_dir, environment.filename) - if force_generate: - _warn_on_ignored_requirements(base_dir, environment.filename) + ce = RSConnectExecutor(**kwargs).validate_server().validate_app_mode(app_mode=app_mode) + ce.make_bundle( + make_voila_source_bundle, + path, + environment, + extra_files, + image=image, + ).deploy_bundle().save_deployed_info().emit_task_log() + else: + if force_generate: + _warn_on_ignored_requirements(base_dir, environment.filename) - ce = RSConnectExecutor(**kwargs).validate_server().validate_app_mode(app_mode=app_mode) - ce.make_bundle( - make_voila_source_bundle, - file, - environment, - extra_files, - image=image, - ).deploy_bundle().save_deployed_info().emit_task_log() + ce = RSConnectExecutor(**kwargs).validate_server().validate_app_mode(app_mode=app_mode) + ce.make_bundle( + make_voila_bundle, + path, + entrypoint, + extra_files, + excludes, + environment, + image=image, + ).deploy_bundle().save_deployed_info().emit_task_log() # noinspection SpellCheckingInspection,DuplicatedCode From 0671dd0419a0ed8f63da6e08845bfc5cf2e41e3f Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Fri, 26 Aug 2022 16:18:45 -0400 Subject: [PATCH 06/97] update return type --- rsconnect/bundle.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rsconnect/bundle.py b/rsconnect/bundle.py index ac934352..9f641c4a 100644 --- a/rsconnect/bundle.py +++ b/rsconnect/bundle.py @@ -808,7 +808,7 @@ def pack_relevant_files( entrypoint: str, extra_files: typing.List[str], excludes: typing.List[str], -) -> typing.Tuple[typing.Dict[str, typing.Any], typing.List[str]]: +) -> typing.List[str]: """ Packs all the extra files and filters out excludes. From f23dfe8bbca33e65154c57b23ee17653343bbc40 Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Fri, 26 Aug 2022 17:28:45 -0400 Subject: [PATCH 07/97] update voila directory deploy to use single code path --- rsconnect/bundle.py | 4 ++++ rsconnect/main.py | 48 ++++++++++++++++----------------------------- 2 files changed, 21 insertions(+), 31 deletions(-) diff --git a/rsconnect/bundle.py b/rsconnect/bundle.py index 9f641c4a..82a6065a 100644 --- a/rsconnect/bundle.py +++ b/rsconnect/bundle.py @@ -887,6 +887,8 @@ def make_voila_bundle( """ mimetypes.add_type("text/ipynb", ".ipynb") entrypoint = entrypoint or infer_entrypoint(path=path, mimetype="text/ipynb") + if extra_files is None: + extra_files = [] base_dir = dirname(entrypoint) nb_name = basename(entrypoint) @@ -905,6 +907,8 @@ def make_voila_bundle( with tarfile.open(mode="w:gz", fileobj=bundle_file) as bundle: bundle_add_buffer(bundle, "manifest.json", json.dumps(manifest, indent=2)) + if isfile(path): + bundle_add_file(bundle, environment.filename, base_dir) for rel_path in relevant_files: bundle_add_file(bundle, rel_path, path) diff --git a/rsconnect/main.py b/rsconnect/main.py index dfbc7e94..b46dc011 100644 --- a/rsconnect/main.py +++ b/rsconnect/main.py @@ -800,38 +800,24 @@ def deploy_voila( set_verbosity(verbose) app_mode = AppModes.JUPYTER_VOILA python, environment = get_python_env_info(path, python, conda_mode=False, force_generate=force_generate) - if isfile(path): - kwargs["extra_files"] = extra_files = validate_extra_files(dirname(path), extra_files) - base_dir = dirname(path) - _warn_on_ignored_manifest(base_dir) - _warn_if_no_requirements_file(base_dir) - _warn_if_environment_directory(base_dir) - - if force_generate: - _warn_on_ignored_requirements(base_dir, environment.filename) - - ce = RSConnectExecutor(**kwargs).validate_server().validate_app_mode(app_mode=app_mode) - ce.make_bundle( - make_voila_source_bundle, - path, - environment, - extra_files, - image=image, - ).deploy_bundle().save_deployed_info().emit_task_log() - else: - if force_generate: - _warn_on_ignored_requirements(base_dir, environment.filename) + kwargs["extra_files"] = extra_files = validate_extra_files(dirname(path), extra_files) + base_dir = dirname(path) + _warn_on_ignored_manifest(base_dir) + _warn_if_no_requirements_file(base_dir) + _warn_if_environment_directory(base_dir) + if force_generate: + _warn_on_ignored_requirements(base_dir, environment.filename) - ce = RSConnectExecutor(**kwargs).validate_server().validate_app_mode(app_mode=app_mode) - ce.make_bundle( - make_voila_bundle, - path, - entrypoint, - extra_files, - excludes, - environment, - image=image, - ).deploy_bundle().save_deployed_info().emit_task_log() + ce = RSConnectExecutor(**kwargs).validate_server().validate_app_mode(app_mode=app_mode) + ce.make_bundle( + make_voila_bundle, + path, + entrypoint, + extra_files, + excludes, + environment, + image=image, + ).deploy_bundle().save_deployed_info().emit_task_log() # noinspection SpellCheckingInspection,DuplicatedCode From 4914f824465b57c8e922d5a7eedf6c7e43f281db Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Fri, 26 Aug 2022 17:31:42 -0400 Subject: [PATCH 08/97] delete unused code path --- rsconnect/bundle.py | 43 ------------------------------------------- rsconnect/main.py | 1 - 2 files changed, 44 deletions(-) diff --git a/rsconnect/bundle.py b/rsconnect/bundle.py index 82a6065a..c4fe7197 100644 --- a/rsconnect/bundle.py +++ b/rsconnect/bundle.py @@ -318,49 +318,6 @@ def make_notebook_source_bundle( return bundle_file -def make_voila_source_bundle( - file: str, - environment: Environment, - extra_files: typing.List[str], - image: str = None, -) -> typing.IO[bytes]: - """Create a bundle containing the specified notebook and python environment. - - Returns a file-like object containing the bundle tarball. - """ - if extra_files is None: - extra_files = [] - base_dir = dirname(file) - nb_name = basename(file) - - manifest = make_source_manifest(AppModes.JUPYTER_VOILA, environment, nb_name, None, image) - manifest_add_file(manifest, nb_name, base_dir) - manifest_add_buffer(manifest, environment.filename, environment.contents) - - if extra_files: - skip = [nb_name, environment.filename, "manifest.json"] - extra_files = sorted(list(set(extra_files) - set(skip))) - - for rel_path in extra_files: - manifest_add_file(manifest, rel_path, base_dir) - - logger.debug("manifest: %r", manifest) - - bundle_file = tempfile.TemporaryFile(prefix="rsc_bundle") - with tarfile.open(mode="w:gz", fileobj=bundle_file) as bundle: - - # add the manifest first in case we want to partially untar the bundle for inspection - bundle_add_buffer(bundle, "manifest.json", json.dumps(manifest, indent=2)) - bundle_add_buffer(bundle, environment.filename, environment.contents) - bundle_add_file(bundle, nb_name, base_dir) - - for rel_path in extra_files: - bundle_add_file(bundle, rel_path, base_dir) - - bundle_file.seek(0) - return bundle_file - - def make_quarto_source_bundle( file_or_directory: str, inspect: typing.Dict[str, typing.Any], diff --git a/rsconnect/main.py b/rsconnect/main.py index b46dc011..e48b2507 100644 --- a/rsconnect/main.py +++ b/rsconnect/main.py @@ -48,7 +48,6 @@ make_notebook_html_bundle, make_notebook_source_bundle, make_voila_bundle, - make_voila_source_bundle, read_manifest_app_mode, write_notebook_manifest_json, write_api_manifest_json, From b58cb34ec5235811bb0b1e856338288ed4a8b0d1 Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Fri, 26 Aug 2022 17:34:32 -0400 Subject: [PATCH 09/97] update imports --- rsconnect/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rsconnect/main.py b/rsconnect/main.py index e48b2507..922aca9e 100644 --- a/rsconnect/main.py +++ b/rsconnect/main.py @@ -7,7 +7,7 @@ import textwrap import click from six import text_type -from os.path import abspath, dirname, exists, isdir, join, isfile +from os.path import abspath, dirname, exists, isdir, join from functools import wraps from .environment import EnvironmentException from .exception import RSConnectException From 6fdc81126a1c3a70c7dfd182f8471cfb54f91696 Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Mon, 29 Aug 2022 13:11:48 -0400 Subject: [PATCH 10/97] raise entrypoint inference error --- rsconnect/bundle.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/rsconnect/bundle.py b/rsconnect/bundle.py index c4fe7197..7bae6158 100644 --- a/rsconnect/bundle.py +++ b/rsconnect/bundle.py @@ -725,7 +725,12 @@ def infer_entrypoint(path, mimetype): mimetype_filelist[guess_type(file)[0]].append(rel_path) if file in default_mimetype_entrypoints[mimetype]: return file - return mimetype_filelist[mimetype].pop() if len(mimetype_filelist[mimetype]) == 1 else None + res = mimetype_filelist[mimetype].pop() if len(mimetype_filelist[mimetype]) == 1 else None + if not res: + raise RuntimeError( + "Unable to infer entrypoint. Please provide an explicit entrypoint, or ensure that the provided path contains exactly one valid content type that can function as the entrypoint." + ) + return res def make_html_bundle( From b3aa3c3a4e5ea046153a64e55146d2ad120990e2 Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Mon, 29 Aug 2022 14:14:17 -0400 Subject: [PATCH 11/97] rename pack_relevant_files to pack_extra_files --- rsconnect/bundle.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/rsconnect/bundle.py b/rsconnect/bundle.py index 7bae6158..c5ece421 100644 --- a/rsconnect/bundle.py +++ b/rsconnect/bundle.py @@ -765,7 +765,7 @@ def make_html_bundle( return bundle_file -def pack_relevant_files( +def pack_extra_files( path: str, entrypoint: str, extra_files: typing.List[str], @@ -861,8 +861,8 @@ def make_voila_bundle( if extra_files: skip = [nb_name, environment.filename, "manifest.json"] extra_files = sorted(list(set(extra_files) - set(skip))) - relevant_files = pack_relevant_files(path, entrypoint, extra_files, excludes) - for rel_path in relevant_files: + packed_extra_files = pack_extra_files(path, entrypoint, extra_files, excludes) + for rel_path in packed_extra_files: manifest_add_file(manifest, rel_path, path) bundle_file = tempfile.TemporaryFile(prefix="rsc_bundle") @@ -872,7 +872,7 @@ def make_voila_bundle( if isfile(path): bundle_add_file(bundle, environment.filename, base_dir) - for rel_path in relevant_files: + for rel_path in packed_extra_files: bundle_add_file(bundle, rel_path, path) # rewind file pointer From 6ef8ae964e1315952b9638d04901c86c3f52c49a Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Mon, 29 Aug 2022 14:43:22 -0400 Subject: [PATCH 12/97] add --disable-pip-version-check to pip list --- rsconnect/environment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rsconnect/environment.py b/rsconnect/environment.py index a2851b3b..2be6d64c 100644 --- a/rsconnect/environment.py +++ b/rsconnect/environment.py @@ -218,7 +218,7 @@ def pip_freeze(): """ try: proc = subprocess.Popen( - [sys.executable, "-m", "pip", "list", "--format=freeze"], + [sys.executable, "-m", "pip", "list", "--format=freeze", "--disable-pip-version-check"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, From e50d94c0c362a47d0cf114ef4a81a4da3490a35f Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Mon, 29 Aug 2022 15:53:40 -0400 Subject: [PATCH 13/97] add defaults to write_voila_manifest_json --- rsconnect/bundle.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rsconnect/bundle.py b/rsconnect/bundle.py index c5ece421..0bf9145c 100644 --- a/rsconnect/bundle.py +++ b/rsconnect/bundle.py @@ -1390,8 +1390,8 @@ def write_notebook_manifest_json( def write_voila_manifest_json( entry_point_file: str, environment: Environment, - app_mode: AppMode, - extra_files: typing.List[str], + app_mode: AppMode = AppModes.JUPYTER_VOILA, + extra_files: typing.List[str] = None, image: str = None, ) -> bool: """ From 564e82a7e4de5d5cb4a97ddcf4167a355bb7ae86 Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Mon, 29 Aug 2022 15:54:29 -0400 Subject: [PATCH 14/97] generate voila requirements when none exists --- rsconnect/bundle.py | 10 +++++++++- rsconnect/main.py | 3 ++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/rsconnect/bundle.py b/rsconnect/bundle.py index 0bf9145c..16c60810 100644 --- a/rsconnect/bundle.py +++ b/rsconnect/bundle.py @@ -834,16 +834,22 @@ def make_voila_bundle( entrypoint: str, extra_files: typing.List[str], excludes: typing.List[str], + force_generate: bool, environment: Environment, image: str = None, ) -> typing.IO[bytes]: """ - Create an voila bundle, given a path and a manifest. + Create an voila bundle, given a path and/or entrypoint. + + The bundle contains a manifest.json file created for the given notebook entrypoint file. + If the related environment file (requirements.txt) doesn't + exist (or force_generate is set to True), the environment file will also be written. :param path: the file, or the directory containing the files to deploy. :param entry_point: the main entry point for the API. :param extra_files: a sequence of any extra files to include in the bundle. :param excludes: a sequence of glob patterns that will exclude matched files. + :param force_generate: bool indicating whether to force generate manifest and related environment files. :param image: the optional docker image to be specified for off-host execution. Default = None. :return: a file-like object containing the bundle tarball. """ @@ -854,6 +860,8 @@ def make_voila_bundle( base_dir = dirname(entrypoint) nb_name = basename(entrypoint) + if not exists(join(base_dir, environment.filename)) or force_generate: + write_environment_file(environment, base_dir) manifest = make_source_manifest(AppModes.JUPYTER_VOILA, environment, nb_name, None, image) manifest_add_file(manifest, nb_name, base_dir) manifest_add_buffer(manifest, environment.filename, environment.contents) diff --git a/rsconnect/main.py b/rsconnect/main.py index 922aca9e..2cd32391 100644 --- a/rsconnect/main.py +++ b/rsconnect/main.py @@ -776,7 +776,6 @@ def deploy_notebook( ) @cli_exception_handler def deploy_voila( - connect_server: api.RSConnectServer = None, path: str = None, entrypoint: str = None, python=None, @@ -794,6 +793,7 @@ def deploy_voila( api_key: str = None, insecure: bool = False, cacert: typing.IO = None, + connect_server: api.RSConnectServer = None, ): kwargs = locals() set_verbosity(verbose) @@ -814,6 +814,7 @@ def deploy_voila( entrypoint, extra_files, excludes, + force_generate, environment, image=image, ).deploy_bundle().save_deployed_info().emit_task_log() From 360469c3871159fdbfaccd0431ae74416ffc0c12 Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Mon, 29 Aug 2022 16:00:09 -0400 Subject: [PATCH 15/97] reduce error massage length for E501 --- rsconnect/bundle.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rsconnect/bundle.py b/rsconnect/bundle.py index 16c60810..1402b3a9 100644 --- a/rsconnect/bundle.py +++ b/rsconnect/bundle.py @@ -728,7 +728,7 @@ def infer_entrypoint(path, mimetype): res = mimetype_filelist[mimetype].pop() if len(mimetype_filelist[mimetype]) == 1 else None if not res: raise RuntimeError( - "Unable to infer entrypoint. Please provide an explicit entrypoint, or ensure that the provided path contains exactly one valid content type that can function as the entrypoint." + "Unable to infer entry point. Provide an explicit entry point, or ensure path contains exactly one valid content." ) return res From 1477f9d9cba87ed4487a5e43da54a406f71e867d Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Mon, 29 Aug 2022 16:05:25 -0400 Subject: [PATCH 16/97] shorten error msg further --- rsconnect/bundle.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rsconnect/bundle.py b/rsconnect/bundle.py index 1402b3a9..8bcde58e 100644 --- a/rsconnect/bundle.py +++ b/rsconnect/bundle.py @@ -728,7 +728,7 @@ def infer_entrypoint(path, mimetype): res = mimetype_filelist[mimetype].pop() if len(mimetype_filelist[mimetype]) == 1 else None if not res: raise RuntimeError( - "Unable to infer entry point. Provide an explicit entry point, or ensure path contains exactly one valid content." + "Unable to infer entry point. Provide an entry point, or ensure path contains exactly one valid content." ) return res From 0eab04a50db92c506c2e65f317b0400bd5b9491e Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Mon, 29 Aug 2022 16:27:17 -0400 Subject: [PATCH 17/97] update AppMode description for JUPYTER_VOILA --- rsconnect/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rsconnect/models.py b/rsconnect/models.py index 6e29182d..0c67d562 100644 --- a/rsconnect/models.py +++ b/rsconnect/models.py @@ -78,7 +78,7 @@ class AppModes(object): SHINY_QUARTO = AppMode(13, "quarto-shiny", "Shiny Quarto Document") STATIC_QUARTO = AppMode(14, "quarto-static", "Quarto Document", ".qmd") PYTHON_SHINY = AppMode(15, "python-shiny", "Python Shiny Application") - JUPYTER_VOILA = AppMode(16, "jupyter-voila", "Python Shiny Application") + JUPYTER_VOILA = AppMode(16, "jupyter-voila", "Jupyter Voila Application") _modes = [ UNKNOWN, From 6e4283b76192c75cabd327b7d989c50a87fe219a Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Thu, 1 Sep 2022 13:50:49 -0400 Subject: [PATCH 18/97] simplify packing steps --- rsconnect/bundle.py | 23 +++++------------------ 1 file changed, 5 insertions(+), 18 deletions(-) diff --git a/rsconnect/bundle.py b/rsconnect/bundle.py index 8bcde58e..db6e72b7 100644 --- a/rsconnect/bundle.py +++ b/rsconnect/bundle.py @@ -767,7 +767,6 @@ def make_html_bundle( def pack_extra_files( path: str, - entrypoint: str, extra_files: typing.List[str], excludes: typing.List[str], ) -> typing.List[str]: @@ -782,24 +781,17 @@ def pack_extra_files( :return: a list of the files involved. """ extra_files = list(extra_files) if extra_files else [] + excludes = list(excludes) if excludes else [] if path.startswith(os.curdir): path = relpath(path) - if entrypoint.startswith(os.curdir): - entrypoint = relpath(entrypoint) extra_files = [relpath(f) if isfile(f) and f.startswith(os.curdir) else f for f in extra_files] + # exclude environment directories and manifest if is_environment_dir(path): - excludes = list(excludes or []) + ["bin/", "lib/"] - - extra_files = extra_files or [] - skip = ["manifest.json"] - extra_files = sorted(list(set(extra_files) - set(skip))) - - # Don't include these top-level files. - excludes = list(excludes) if excludes else [] + excludes += ["bin/", "lib/"] excludes.append("manifest.json") - if not isfile(path): + if isdir(path): excludes.extend(list_environment_dirs(path)) glob_set = create_glob_set(path, excludes) @@ -820,13 +812,8 @@ def pack_extra_files( rel_path in extra_files or not glob_set.matches(abs_path) ): file_list.append(rel_path) - # Don't add extra files more than once. - if rel_path in extra_files: - extra_files.remove(rel_path) - - relevant_files = sorted(file_list) - return relevant_files + return sorted(set(file_list)) def make_voila_bundle( From 26be70b66f775c5d07336f71c2d5e08195a0bfa0 Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Thu, 1 Sep 2022 15:34:51 -0400 Subject: [PATCH 19/97] include voila.json automatically --- rsconnect/bundle.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/rsconnect/bundle.py b/rsconnect/bundle.py index db6e72b7..eee1b76e 100644 --- a/rsconnect/bundle.py +++ b/rsconnect/bundle.py @@ -841,9 +841,11 @@ def make_voila_bundle( :return: a file-like object containing the bundle tarball. """ mimetypes.add_type("text/ipynb", ".ipynb") + voila_config = "voila.json" + extra_files = list(extra_files) if extra_files else [] + excludes = list(excludes) if excludes else [] + entrypoint = entrypoint or infer_entrypoint(path=path, mimetype="text/ipynb") - if extra_files is None: - extra_files = [] base_dir = dirname(entrypoint) nb_name = basename(entrypoint) @@ -853,17 +855,16 @@ def make_voila_bundle( manifest_add_file(manifest, nb_name, base_dir) manifest_add_buffer(manifest, environment.filename, environment.contents) - if extra_files: - skip = [nb_name, environment.filename, "manifest.json"] - extra_files = sorted(list(set(extra_files) - set(skip))) - packed_extra_files = pack_extra_files(path, entrypoint, extra_files, excludes) + excludes.extend(["manifest.json", voila_config]) + packed_extra_files = pack_extra_files(path, extra_files, excludes) for rel_path in packed_extra_files: manifest_add_file(manifest, rel_path, path) - bundle_file = tempfile.TemporaryFile(prefix="rsc_bundle") with tarfile.open(mode="w:gz", fileobj=bundle_file) as bundle: bundle_add_buffer(bundle, "manifest.json", json.dumps(manifest, indent=2)) + if exists(join(base_dir, voila_config)): + bundle_add_file(bundle, voila_config, base_dir) if isfile(path): bundle_add_file(bundle, environment.filename, base_dir) From 8725e9fe8486d6743177c47dce5a881c03b33f09 Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Fri, 9 Sep 2022 12:07:22 -0400 Subject: [PATCH 20/97] correct description of deploy voila --- rsconnect/main.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/rsconnect/main.py b/rsconnect/main.py index 2cd32391..b4c878ec 100644 --- a/rsconnect/main.py +++ b/rsconnect/main.py @@ -738,12 +738,8 @@ def deploy_notebook( # noinspection SpellCheckingInspection,DuplicatedCode @deploy.command( name="voila", - short_help="Deploy Voila notebook to RStudio Connect [v2022.09.0+].", - help=( - "Deploy a Voila notebook to RStudio Connect. This may be done by source or as a static HTML " - "page. If the notebook is deployed as a static HTML page (--static), it cannot be scheduled or " - "rerun on the Connect server." - ), + short_help="Deploy Jupyter notebook in Voila mode to RStudio Connect [v2022.09.0+].", + help=("Deploy a Jupyter notebook in Voila mode to RStudio Connect."), ) @server_args @content_args From 0abfe7581a3e537cc7fc393fbe55e31f5ff7b241 Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Tue, 13 Sep 2022 13:31:51 -0400 Subject: [PATCH 21/97] remove infer_entrypoint error check --- rsconnect/bundle.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/rsconnect/bundle.py b/rsconnect/bundle.py index eee1b76e..93145b3f 100644 --- a/rsconnect/bundle.py +++ b/rsconnect/bundle.py @@ -656,6 +656,10 @@ def make_html_bundle_content( """ extra_files = list(extra_files) if extra_files else [] entrypoint = entrypoint or infer_entrypoint(path=path, mimetype="text/html") + if not entrypoint: + raise RSConnectException( + "Unable to infer entry point. Provide an entry point, or ensure path contains exactly one valid content." + ) if path.startswith(os.curdir): path = relpath(path) @@ -726,10 +730,6 @@ def infer_entrypoint(path, mimetype): if file in default_mimetype_entrypoints[mimetype]: return file res = mimetype_filelist[mimetype].pop() if len(mimetype_filelist[mimetype]) == 1 else None - if not res: - raise RuntimeError( - "Unable to infer entry point. Provide an entry point, or ensure path contains exactly one valid content." - ) return res From f0f55977742c24d7a179ad74ab7746bd30f6491c Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Tue, 13 Sep 2022 16:10:04 -0400 Subject: [PATCH 22/97] update write_voila_manifest_json to handle both file and directory --- rsconnect/bundle.py | 51 ++++++++++++++++++++++++++------------------- 1 file changed, 29 insertions(+), 22 deletions(-) diff --git a/rsconnect/bundle.py b/rsconnect/bundle.py index 93145b3f..f3d8da74 100644 --- a/rsconnect/bundle.py +++ b/rsconnect/bundle.py @@ -1384,49 +1384,56 @@ def write_notebook_manifest_json( def write_voila_manifest_json( - entry_point_file: str, + path: str, + entrypoint: str, environment: Environment, app_mode: AppMode = AppModes.JUPYTER_VOILA, extra_files: typing.List[str] = None, + excludes: typing.List[str] = None, + force_generate: bool = True, image: str = None, ) -> bool: """ - Creates and writes a manifest.json file for the given entry point file. If - the application mode is not provided, an attempt will be made to resolve one - based on the extension portion of the entry point file. + Creates and writes a manifest.json file for the given path. - :param entry_point_file: the entry point file (Voila notebook, etc.) to build - the manifest for. + :param path: the file, or the directory containing the files to deploy. + :param entry_point: the main entry point for the API. :param environment: the Python environment to start with. This should be what's returned by the inspect_environment() function. :param app_mode: the application mode to assume. If this is None, the extension portion of the entry point file name will be used to derive one. Previous default = None. :param extra_files: any extra files that should be included in the manifest. Previous default = None. + :param excludes: a sequence of glob patterns that will exclude matched files. + :param force_generate: bool indicating whether to force generate manifest and related environment files. :param image: the optional docker image to be specified for off-host execution. Default = None. - :return: whether or not the environment file (requirements.txt, environment.yml, - etc.) that goes along with the manifest exists. + :return: whether the manifest was written. """ - extra_files = validate_extra_files(dirname(entry_point_file), extra_files) - directory = dirname(entry_point_file) - file_name = basename(entry_point_file) - manifest_path = join(directory, "manifest.json") + mimetypes.add_type("text/ipynb", ".ipynb") + extra_files = list(extra_files) if extra_files else [] + excludes = list(excludes) if excludes else [] - if app_mode is None: - _, extension = splitext(file_name) - app_mode = AppModes.get_by_extension(extension, True) - if app_mode == AppModes.UNKNOWN: - raise RSConnectException('Could not determine the app mode from "%s"; please specify one.' % extension) + entrypoint = entrypoint or infer_entrypoint(path=path, mimetype="text/ipynb") + manifest_data = make_source_manifest(AppModes.JUPYTER_VOILA, environment, entrypoint, None, image) + base_dir = dirname(entrypoint) - manifest_data = make_source_manifest(app_mode, environment, file_name, None, image) - manifest_add_file(manifest_data, file_name, directory) + if isfile(path): + validate_file_is_notebook(entrypoint) + manifest_add_file(manifest_data, entrypoint, base_dir) + + # handle environment files + if not exists(join(base_dir, environment.filename)) or force_generate: + write_environment_file(environment, base_dir) manifest_add_buffer(manifest_data, environment.filename, environment.contents) - for rel_path in extra_files: - manifest_add_file(manifest_data, rel_path, directory) + excludes.extend(["manifest.json"]) + file_list = create_file_list(path, extra_files, excludes) + for rel_path in file_list: + manifest_add_file(manifest_data, rel_path, path) + manifest_path = join(base_dir, "manifest.json") write_manifest_json(manifest_path, manifest_data) - return exists(join(directory, environment.filename)) + return exists(manifest_path) def create_api_manifest_and_environment_file( From 4aedbafbb3701a77dafbfe34719cb3d56a58d873 Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Tue, 13 Sep 2022 16:40:52 -0400 Subject: [PATCH 23/97] rename pack_extra_files to create_file_list --- rsconnect/bundle.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rsconnect/bundle.py b/rsconnect/bundle.py index f3d8da74..a3de1672 100644 --- a/rsconnect/bundle.py +++ b/rsconnect/bundle.py @@ -765,14 +765,14 @@ def make_html_bundle( return bundle_file -def pack_extra_files( +def create_file_list( path: str, extra_files: typing.List[str], excludes: typing.List[str], ) -> typing.List[str]: """ - Packs all the extra files and filters out excludes. + Create a file list from extras and excludes. Excludes any existing manifest.json. :param path: the file, or the directory containing the files to deploy. :param entry_point: the main entry point for the API. From c4f68879bebec7444328fefefa12ced73659a5a8 Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Tue, 13 Sep 2022 16:42:31 -0400 Subject: [PATCH 24/97] update make_voila_bundle --- rsconnect/bundle.py | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/rsconnect/bundle.py b/rsconnect/bundle.py index a3de1672..4741506a 100644 --- a/rsconnect/bundle.py +++ b/rsconnect/bundle.py @@ -833,7 +833,7 @@ def make_voila_bundle( exist (or force_generate is set to True), the environment file will also be written. :param path: the file, or the directory containing the files to deploy. - :param entry_point: the main entry point for the API. + :param entry_point: the main entry point. :param extra_files: a sequence of any extra files to include in the bundle. :param excludes: a sequence of glob patterns that will exclude matched files. :param force_generate: bool indicating whether to force generate manifest and related environment files. @@ -846,29 +846,31 @@ def make_voila_bundle( excludes = list(excludes) if excludes else [] entrypoint = entrypoint or infer_entrypoint(path=path, mimetype="text/ipynb") + manifest_data = make_source_manifest(AppModes.JUPYTER_VOILA, environment, entrypoint, None, image) base_dir = dirname(entrypoint) - nb_name = basename(entrypoint) + if isfile(path): + validate_file_is_notebook(entrypoint) + manifest_add_file(manifest_data, entrypoint, base_dir) + + # handle environment files if not exists(join(base_dir, environment.filename)) or force_generate: write_environment_file(environment, base_dir) - manifest = make_source_manifest(AppModes.JUPYTER_VOILA, environment, nb_name, None, image) - manifest_add_file(manifest, nb_name, base_dir) - manifest_add_buffer(manifest, environment.filename, environment.contents) + manifest_add_buffer(manifest_data, environment.filename, environment.contents) - excludes.extend(["manifest.json", voila_config]) - packed_extra_files = pack_extra_files(path, extra_files, excludes) - for rel_path in packed_extra_files: - manifest_add_file(manifest, rel_path, path) - bundle_file = tempfile.TemporaryFile(prefix="rsc_bundle") + excludes.extend(["manifest.json"]) + file_list = create_file_list(path, extra_files, excludes) + for rel_path in file_list: + manifest_add_file(manifest_data, rel_path, path) + bundle_file = tempfile.TemporaryFile(prefix="rsc_bundle") with tarfile.open(mode="w:gz", fileobj=bundle_file) as bundle: - bundle_add_buffer(bundle, "manifest.json", json.dumps(manifest, indent=2)) + bundle_add_buffer(bundle, "manifest.json", json.dumps(manifest_data, indent=2)) if exists(join(base_dir, voila_config)): bundle_add_file(bundle, voila_config, base_dir) if isfile(path): bundle_add_file(bundle, environment.filename, base_dir) - - for rel_path in packed_extra_files: + for rel_path in file_list: bundle_add_file(bundle, rel_path, path) # rewind file pointer From e0cfe36dda411e46f4189ae196b58ff474befca5 Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Tue, 13 Sep 2022 16:44:23 -0400 Subject: [PATCH 25/97] update write_manifest_voila --- rsconnect/main.py | 54 +++++++++++++++++++++++++++-------------------- 1 file changed, 31 insertions(+), 23 deletions(-) diff --git a/rsconnect/main.py b/rsconnect/main.py index b4c878ec..501ab22f 100644 --- a/rsconnect/main.py +++ b/rsconnect/main.py @@ -738,7 +738,7 @@ def deploy_notebook( # noinspection SpellCheckingInspection,DuplicatedCode @deploy.command( name="voila", - short_help="Deploy Jupyter notebook in Voila mode to RStudio Connect [v2022.09.0+].", + short_help="Deploy Jupyter notebook in Voila mode to RStudio Connect [v2022.10.0+].", help=("Deploy a Jupyter notebook in Voila mode to RStudio Connect."), ) @server_args @@ -1347,13 +1347,6 @@ def write_manifest_notebook( help="Path to Python interpreter whose environment should be used. " + "The Python environment must have the rsconnect package installed.", ) -@click.option( - "--conda", - "-C", - is_flag=True, - hidden=True, - help="Use Conda to deploy (requires RStudio Connect version 1.8.2 or later)", -) @click.option( "--force-generate", "-g", @@ -1367,27 +1360,37 @@ def write_manifest_notebook( help="Target image to be used during content execution (only applicable if the RStudio Connect " "server is configured to use off-host execution)", ) -@click.argument("file", type=click.Path(exists=True, dir_okay=False, file_okay=True)) +@click.argument("path", type=click.Path(exists=True, dir_okay=True, file_okay=True)) @click.argument( "extra_files", nargs=-1, type=click.Path(exists=True, dir_okay=False, file_okay=True), ) +@click.option("--entrypoint", "-e", help=("The module and executable object which serves as the entry point.")) +@click.option( + "--exclude", + "-x", + multiple=True, + help=( + "Specify a glob pattern for ignoring files when building the bundle. Note that your shell may try " + "to expand this which will not do what you expect. Generally, it's safest to quote the pattern. " + "This option may be repeated." + ), +) def write_manifest_voila( + path: str, + entrypoint: str, overwrite, python, - conda, force_generate, verbose, - file, extra_files, + exclude, image, ): set_verbosity(verbose) with cli_feedback("Checking arguments"): - validate_file_is_notebook(file) - - base_dir = dirname(file) + base_dir = dirname(path) extra_files = validate_extra_files(base_dir, extra_files) manifest_path = join(base_dir, "manifest.json") @@ -1395,18 +1398,11 @@ def write_manifest_voila( raise RSConnectException("manifest.json already exists. Use --overwrite to overwrite.") with cli_feedback("Inspecting Python environment"): - python, environment = get_python_env_info(file, python, conda, force_generate) + python, environment = get_python_env_info(path, python, False, force_generate) _warn_on_ignored_conda_env(environment) - with cli_feedback("Creating manifest.json"): - environment_file_exists = write_voila_manifest_json( - file, - environment, - AppModes.JUPYTER_VOILA, - extra_files, - image, - ) + environment_file_exists = exists(join(base_dir, environment.filename)) if environment_file_exists and not force_generate: click.secho( @@ -1417,6 +1413,18 @@ def write_manifest_voila( with cli_feedback("Creating %s" % environment.filename): write_environment_file(environment, base_dir) + with cli_feedback("Creating manifest.json"): + write_voila_manifest_json( + path, + entrypoint, + environment, + AppModes.JUPYTER_VOILA, + extra_files, + exclude, + force_generate, + image, + ) + @write_manifest.command( name="quarto", From 8f2197322ec61fe72f2fed63cfd15e2b45080da5 Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Wed, 18 Jan 2023 12:47:11 -0500 Subject: [PATCH 26/97] delete old version of create_file_list --- rsconnect/bundle.py | 51 --------------------------------------------- 1 file changed, 51 deletions(-) diff --git a/rsconnect/bundle.py b/rsconnect/bundle.py index 9b154b13..cd91d9dc 100644 --- a/rsconnect/bundle.py +++ b/rsconnect/bundle.py @@ -748,57 +748,6 @@ def make_html_bundle( return bundle_file -def create_file_list( - path: str, - extra_files: typing.List[str], - excludes: typing.List[str], -) -> typing.List[str]: - - """ - Create a file list from extras and excludes. Excludes any existing manifest.json. - - :param path: the file, or the directory containing the files to deploy. - :param entry_point: the main entry point for the API. - :param extra_files: a sequence of any extra files to include in the bundle. - :param excludes: a sequence of glob patterns that will exclude matched files. - :return: a list of the files involved. - """ - extra_files = list(extra_files) if extra_files else [] - excludes = list(excludes) if excludes else [] - - if path.startswith(os.curdir): - path = relpath(path) - extra_files = [relpath(f) if isfile(f) and f.startswith(os.curdir) else f for f in extra_files] - - # exclude environment directories and manifest - if is_environment_dir(path): - excludes += ["bin/", "lib/"] - excludes.append("manifest.json") - if isdir(path): - excludes.extend(list_environment_dirs(path)) - glob_set = create_glob_set(path, excludes) - - file_list = [] - - for rel_path in extra_files: - file_list.append(rel_path) - - if isfile(path): - file_list.append(path) - else: - for subdir, dirs, files in os.walk(path): - for file in files: - abs_path = os.path.join(subdir, file) - rel_path = os.path.relpath(abs_path, path) - - if keep_manifest_specified_file(rel_path) and ( - rel_path in extra_files or not glob_set.matches(abs_path) - ): - file_list.append(rel_path) - - return sorted(set(file_list)) - - def make_voila_bundle( path: str, entrypoint: str, From e5c20d260752fb01e7093725c93a99c31a037f51 Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Fri, 20 Jan 2023 18:32:08 -0500 Subject: [PATCH 27/97] specify ipynb mimetype by suffix --- rsconnect/bundle.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/rsconnect/bundle.py b/rsconnect/bundle.py index cd91d9dc..4876c527 100644 --- a/rsconnect/bundle.py +++ b/rsconnect/bundle.py @@ -51,6 +51,8 @@ ] directories_to_ignore = {Path(d) for d in directories_ignore_list} +mimetypes.add_type("text/ipynb", ".ipynb") + # noinspection SpellCheckingInspection def make_source_manifest( From c18b65bef2169034983aaf4e4a68248187b67a9a Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Fri, 20 Jan 2023 18:32:56 -0500 Subject: [PATCH 28/97] add Manifest as a class --- rsconnect/bundle.py | 91 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) diff --git a/rsconnect/bundle.py b/rsconnect/bundle.py index 4876c527..81151822 100644 --- a/rsconnect/bundle.py +++ b/rsconnect/bundle.py @@ -54,6 +54,97 @@ mimetypes.add_type("text/ipynb", ".ipynb") +class Manifest: + def __init__(self, *args, **kwargs) -> None: + self.data = dict() + version = kwargs.get("version") + environment = kwargs.get("environment") + app_mode = kwargs.get("app_mode") + entrypoint = kwargs.get("entrypoint") + quarto_inspection = kwargs.get("quarto_inspection") + environment = kwargs.get("environment") + image = kwargs.get("image") + + self.data["version"] = version if version else 1 + if environment: + self.data["locale"] = environment.locale + + self.data["metadata"] = ( + { + "appmode": app_mode.name(), + } + if app_mode + else { + "appmode": AppModes.UNKNOWN, + } + ) + + if entrypoint: + self.data["metadata"]["entrypoint"] = entrypoint + + if quarto_inspection: + self.data["quarto"] = { + "version": quarto_inspection.get("quarto", {}).get("version", "99.9.9"), + "engines": quarto_inspection.get("engines", []), + } + project_config = quarto_inspection.get("config", {}).get("project", {}) + render_targets = project_config.get("render", []) + if len(render_targets): + self.data["metadata"]["primary_rmd"] = render_targets[0] + project_type = project_config.get("type", None) + if project_type or len(render_targets) > 1: + self.data["metadata"]["content_category"] = "site" + + if environment: + package_manager = environment.package_manager + self.data["python"] = { + "version": environment.python, + "package_manager": { + "name": package_manager, + "version": getattr(environment, package_manager), + "package_file": environment.filename, + }, + } + + if image: + self.data["environment"] = { + "image": image, + } + + self.data["files"] = {} + + @classmethod + def from_json(cls, json_str): + return cls(json.loads(json_str)) + + @classmethod + def from_json_file(cls, json_path): + with open(json_path) as json_file: + return cls(json.load(json_file)) + + def json(self): + return json.dumps(self.data) + + @property + def entrypoint(self): + if "entrypoint" in self.data: + return self.data["entrypoint"] + return None + + @entrypoint.setter + def entrypoint(self, value): + self.data["entrypoint"] = value + + def add_file(self, path): + self.data["files"][path] = {"checksum": file_checksum(path)} + return self + + def discard_file(self, path): + if path in self.data["files"]: + del self.data["files"][path] + return self + + # noinspection SpellCheckingInspection def make_source_manifest( app_mode: AppMode, From a3f52e734cec3574743bae8d04d0007f42a71ccb Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Fri, 20 Jan 2023 18:33:15 -0500 Subject: [PATCH 29/97] add Bundle as a class --- rsconnect/bundle.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/rsconnect/bundle.py b/rsconnect/bundle.py index 81151822..39b7653f 100644 --- a/rsconnect/bundle.py +++ b/rsconnect/bundle.py @@ -145,6 +145,25 @@ def discard_file(self, path): return self +class Bundle: + def __init__(self, *args, **kwargs) -> None: + self.file_locations = set() + + def add_file(self, filepath): + self.file_locations.add(filepath) + + def discard_file(self, filepath): + self.file_locations.discard(filepath) + + def to_file(self): + bundle_file = tempfile.TemporaryFile(prefix="rsc_bundle") + with tarfile.open(mode="w:gz", fileobj=bundle_file) as bundle: + for file in self.file_locations: + bundle.add(file) + bundle_file.seek(0) + return bundle_file + + # noinspection SpellCheckingInspection def make_source_manifest( app_mode: AppMode, From 76a3e61112275872a82008cf252304c3cc517a68 Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Fri, 20 Jan 2023 18:40:25 -0500 Subject: [PATCH 30/97] create predecessor to write_voila_manifest_json --- rsconnect/bundle.py | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/rsconnect/bundle.py b/rsconnect/bundle.py index 39b7653f..3268d75c 100644 --- a/rsconnect/bundle.py +++ b/rsconnect/bundle.py @@ -1417,7 +1417,7 @@ def write_notebook_manifest_json( return exists(join(directory, environment.filename)) -def write_voila_manifest_json( +def create_voila_manifest_json( path: str, entrypoint: str, environment: Environment, @@ -1426,7 +1426,7 @@ def write_voila_manifest_json( excludes: typing.List[str] = None, force_generate: bool = True, image: str = None, -) -> bool: +) -> Manifest: """ Creates and writes a manifest.json file for the given path. @@ -1440,34 +1440,30 @@ def write_voila_manifest_json( :param excludes: a sequence of glob patterns that will exclude matched files. :param force_generate: bool indicating whether to force generate manifest and related environment files. :param image: the optional docker image to be specified for off-host execution. Default = None. - :return: whether the manifest was written. + :return: the manifest data structure. """ - mimetypes.add_type("text/ipynb", ".ipynb") extra_files = list(extra_files) if extra_files else [] excludes = list(excludes) if excludes else [] entrypoint = entrypoint or infer_entrypoint(path=path, mimetype="text/ipynb") - manifest_data = make_source_manifest(AppModes.JUPYTER_VOILA, environment, entrypoint, None, image) base_dir = dirname(entrypoint) + manifest = Manifest(app_mode=AppModes.JUPYTER_VOILA, environment=environment, entrypoint=entrypoint, image=image) if isfile(path): validate_file_is_notebook(entrypoint) - manifest_add_file(manifest_data, entrypoint, base_dir) + manifest.entrypoint = entrypoint # handle environment files if not exists(join(base_dir, environment.filename)) or force_generate: - write_environment_file(environment, base_dir) - manifest_add_buffer(manifest_data, environment.filename, environment.contents) + manifest.add_file(join(base_dir, environment.filename)) excludes.extend(["manifest.json"]) file_list = create_file_list(path, extra_files, excludes) for rel_path in file_list: - manifest_add_file(manifest_data, rel_path, path) + path = join(base_dir, rel_path) if os.path.isdir(base_dir) else rel_path + manifest.add_file(path) - manifest_path = join(base_dir, "manifest.json") - write_manifest_json(manifest_path, manifest_data) - - return exists(manifest_path) + return manifest def create_api_manifest_and_environment_file( From 380f7be5f1c04e1e74395e7a292fe7385b679d2c Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Fri, 20 Jan 2023 18:40:45 -0500 Subject: [PATCH 31/97] refactor write_voila_manifest_json --- rsconnect/bundle.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/rsconnect/bundle.py b/rsconnect/bundle.py index 3268d75c..42425a5a 100644 --- a/rsconnect/bundle.py +++ b/rsconnect/bundle.py @@ -1466,6 +1466,39 @@ def create_voila_manifest_json( return manifest +def write_voila_manifest_json( + path: str, + entrypoint: str, + environment: Environment, + app_mode: AppMode = AppModes.JUPYTER_VOILA, + extra_files: typing.List[str] = None, + excludes: typing.List[str] = None, + force_generate: bool = True, + image: str = None, +) -> bool: + """ + Creates and writes a manifest.json file for the given path. + + :param path: the file, or the directory containing the files to deploy. + :param entry_point: the main entry point for the API. + :param environment: the Python environment to start with. This should be what's + returned by the inspect_environment() function. + :param app_mode: the application mode to assume. If this is None, the extension + portion of the entry point file name will be used to derive one. Previous default = None. + :param extra_files: any extra files that should be included in the manifest. Previous default = None. + :param excludes: a sequence of glob patterns that will exclude matched files. + :param force_generate: bool indicating whether to force generate manifest and related environment files. + :param image: the optional docker image to be specified for off-host execution. Default = None. + :return: whether the manifest was written. + """ + manifest = create_voila_manifest_json(**locals()) + entrypoint = entrypoint or infer_entrypoint(path=path, mimetype="text/ipynb") + base_dir = dirname(entrypoint) + manifest_path = join(base_dir, "manifest.json") + write_manifest_json(manifest_path, manifest.data) + return exists(manifest_path) + + def create_api_manifest_and_environment_file( directory: str, entry_point: str, From 8fa16a3963814602be96a905da37b82d64827c43 Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Fri, 20 Jan 2023 18:45:21 -0500 Subject: [PATCH 32/97] refactor make_voila_bundle --- rsconnect/bundle.py | 41 ++++++++--------------------------------- 1 file changed, 8 insertions(+), 33 deletions(-) diff --git a/rsconnect/bundle.py b/rsconnect/bundle.py index 42425a5a..73a5edd7 100644 --- a/rsconnect/bundle.py +++ b/rsconnect/bundle.py @@ -884,43 +884,18 @@ def make_voila_bundle( :param image: the optional docker image to be specified for off-host execution. Default = None. :return: a file-like object containing the bundle tarball. """ - mimetypes.add_type("text/ipynb", ".ipynb") voila_config = "voila.json" extra_files = list(extra_files) if extra_files else [] - excludes = list(excludes) if excludes else [] - - entrypoint = entrypoint or infer_entrypoint(path=path, mimetype="text/ipynb") - manifest_data = make_source_manifest(AppModes.JUPYTER_VOILA, environment, entrypoint, None, image) - base_dir = dirname(entrypoint) - - if isfile(path): - validate_file_is_notebook(entrypoint) - manifest_add_file(manifest_data, entrypoint, base_dir) - - # handle environment files - if not exists(join(base_dir, environment.filename)) or force_generate: - write_environment_file(environment, base_dir) - manifest_add_buffer(manifest_data, environment.filename, environment.contents) + extra_files.append(voila_config) - excludes.extend(["manifest.json"]) - file_list = create_file_list(path, extra_files, excludes) - for rel_path in file_list: - manifest_add_file(manifest_data, rel_path, path) - - bundle_file = tempfile.TemporaryFile(prefix="rsc_bundle") - with tarfile.open(mode="w:gz", fileobj=bundle_file) as bundle: - bundle_add_buffer(bundle, "manifest.json", json.dumps(manifest_data, indent=2)) - if exists(join(base_dir, voila_config)): - bundle_add_file(bundle, voila_config, base_dir) - if isfile(path): - bundle_add_file(bundle, environment.filename, base_dir) - for rel_path in file_list: - bundle_add_file(bundle, rel_path, path) - - # rewind file pointer - bundle_file.seek(0) + manifest = create_voila_manifest_json(**locals()) + if manifest.get("files") is None: + return - return bundle_file + bundle = Bundle() + for f in manifest["files"]: + bundle.add(f) + return bundle.to_file() def make_api_bundle( From 926a836730b1df38333bba18e1690ff215d1d961 Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Fri, 20 Jan 2023 18:50:27 -0500 Subject: [PATCH 33/97] add type annotation --- rsconnect/bundle.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rsconnect/bundle.py b/rsconnect/bundle.py index 73a5edd7..d9641766 100644 --- a/rsconnect/bundle.py +++ b/rsconnect/bundle.py @@ -147,7 +147,7 @@ def discard_file(self, path): class Bundle: def __init__(self, *args, **kwargs) -> None: - self.file_locations = set() + self.file_locations: set = set() def add_file(self, filepath): self.file_locations.add(filepath) From 2a23e92aeacc1f2c89d6e51f0fdd8809509beefd Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Fri, 20 Jan 2023 18:51:08 -0500 Subject: [PATCH 34/97] correct call to manifest.data --- rsconnect/bundle.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/rsconnect/bundle.py b/rsconnect/bundle.py index d9641766..f7f9a5a1 100644 --- a/rsconnect/bundle.py +++ b/rsconnect/bundle.py @@ -889,12 +889,12 @@ def make_voila_bundle( extra_files.append(voila_config) manifest = create_voila_manifest_json(**locals()) - if manifest.get("files") is None: - return + if manifest.data.get("files") is None: + return None bundle = Bundle() - for f in manifest["files"]: - bundle.add(f) + for f in manifest.data["files"]: + bundle.add_file(f) return bundle.to_file() From 3e2cb44a3e30558755b96457fb9c5c7b5d7f2a8f Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Mon, 23 Jan 2023 18:12:42 -0500 Subject: [PATCH 35/97] entrypoint is in metadata --- rsconnect/bundle.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rsconnect/bundle.py b/rsconnect/bundle.py index f7f9a5a1..ddb5b2b0 100644 --- a/rsconnect/bundle.py +++ b/rsconnect/bundle.py @@ -127,13 +127,13 @@ def json(self): @property def entrypoint(self): - if "entrypoint" in self.data: - return self.data["entrypoint"] + if "entrypoint" in self.data["metadata"]: + return self.data["metadata"]["entrypoint"] return None @entrypoint.setter def entrypoint(self, value): - self.data["entrypoint"] = value + self.data["metadata"]["entrypoint"] = value def add_file(self, path): self.data["files"][path] = {"checksum": file_checksum(path)} From 9c757010c63c57629e18c145d12ae6e0539c5038 Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Mon, 23 Jan 2023 18:13:11 -0500 Subject: [PATCH 36/97] make manifest.json a property --- rsconnect/bundle.py | 1 + 1 file changed, 1 insertion(+) diff --git a/rsconnect/bundle.py b/rsconnect/bundle.py index ddb5b2b0..2de93acd 100644 --- a/rsconnect/bundle.py +++ b/rsconnect/bundle.py @@ -122,6 +122,7 @@ def from_json_file(cls, json_path): with open(json_path) as json_file: return cls(json.load(json_file)) + @property def json(self): return json.dumps(self.data) From 80eeb79b189630e9e8abb94c17644a10c51e77b1 Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Mon, 23 Jan 2023 18:41:19 -0500 Subject: [PATCH 37/97] check metadata when checking entrypoint --- rsconnect/bundle.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/rsconnect/bundle.py b/rsconnect/bundle.py index 2de93acd..674f5e29 100644 --- a/rsconnect/bundle.py +++ b/rsconnect/bundle.py @@ -128,6 +128,8 @@ def json(self): @property def entrypoint(self): + if "metadata" not in self.data: + return None if "entrypoint" in self.data["metadata"]: return self.data["metadata"]["entrypoint"] return None From cd1369f27c0ae8382fee9947026686f04c6fde65 Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Tue, 24 Jan 2023 13:14:09 -0500 Subject: [PATCH 38/97] add file according to relative dir --- rsconnect/bundle.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/rsconnect/bundle.py b/rsconnect/bundle.py index 674f5e29..e581b242 100644 --- a/rsconnect/bundle.py +++ b/rsconnect/bundle.py @@ -139,10 +139,14 @@ def entrypoint(self, value): self.data["metadata"]["entrypoint"] = value def add_file(self, path): + base_dir = dirname(self.entrypoint) + path = join(base_dir, path) if os.path.isdir(self.entrypoint) else path self.data["files"][path] = {"checksum": file_checksum(path)} return self def discard_file(self, path): + base_dir = dirname(self.entrypoint) + path = join(base_dir, path) if os.path.isdir(self.entrypoint) else path if path in self.data["files"]: del self.data["files"][path] return self @@ -891,13 +895,19 @@ def make_voila_bundle( extra_files = list(extra_files) if extra_files else [] extra_files.append(voila_config) - manifest = create_voila_manifest_json(**locals()) + manifest = create_voila_manifest(**locals()) if manifest.data.get("files") is None: return None + entrypoint = entrypoint or infer_entrypoint(path=path, mimetype="text/ipynb") + base_dir = dirname(entrypoint) + manifest_path = join(base_dir, "manifest.json") + write_manifest_json(manifest_path, manifest.data) bundle = Bundle() for f in manifest.data["files"]: bundle.add_file(f) + bundle.add_file(manifest_path) + return bundle.to_file() @@ -1395,7 +1405,7 @@ def write_notebook_manifest_json( return exists(join(directory, environment.filename)) -def create_voila_manifest_json( +def create_voila_manifest( path: str, entrypoint: str, environment: Environment, @@ -1404,6 +1414,7 @@ def create_voila_manifest_json( excludes: typing.List[str] = None, force_generate: bool = True, image: str = None, + **kwargs ) -> Manifest: """ Creates and writes a manifest.json file for the given path. @@ -1469,7 +1480,7 @@ def write_voila_manifest_json( :param image: the optional docker image to be specified for off-host execution. Default = None. :return: whether the manifest was written. """ - manifest = create_voila_manifest_json(**locals()) + manifest = create_voila_manifest(**locals()) entrypoint = entrypoint or infer_entrypoint(path=path, mimetype="text/ipynb") base_dir = dirname(entrypoint) manifest_path = join(base_dir, "manifest.json") From 1b6a3a86d75f0cfbd18da0ae928a97d418b7fbd8 Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Mon, 6 Feb 2023 13:29:32 -0500 Subject: [PATCH 39/97] enable multi-notebook mode via empty string --- rsconnect/bundle.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/rsconnect/bundle.py b/rsconnect/bundle.py index e581b242..e46b16fc 100644 --- a/rsconnect/bundle.py +++ b/rsconnect/bundle.py @@ -1441,6 +1441,8 @@ def create_voila_manifest( if isfile(path): validate_file_is_notebook(entrypoint) manifest.entrypoint = entrypoint + else: + manifest.entrypoint = "" if not entrypoint else entrypoint # multi-notebook mode # handle environment files if not exists(join(base_dir, environment.filename)) or force_generate: From 2007e6dca1f9efa0d5aadc3087c496d102d739cc Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Mon, 6 Feb 2023 19:42:16 -0500 Subject: [PATCH 40/97] unstage existing validate_extra_files --- rsconnect/bundle.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rsconnect/bundle.py b/rsconnect/bundle.py index 8c779f1f..efe2b85a 100644 --- a/rsconnect/bundle.py +++ b/rsconnect/bundle.py @@ -1078,7 +1078,7 @@ def validate_file_is_notebook(file_name): raise RSConnectException("A Jupyter notebook (.ipynb) file is required here.") -def validate_extra_files(directory, extra_files): +def validate_extra_files_(directory, extra_files): """ If the user specified a list of extra files, validate that they all exist and are beneath the given directory and, if so, return a list of them made relative to that From 2220e98e851bf3696f64c466274b47ae92ba92fb Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Mon, 6 Feb 2023 19:43:12 -0500 Subject: [PATCH 41/97] refactor validate_extra_files --- rsconnect/bundle.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/rsconnect/bundle.py b/rsconnect/bundle.py index efe2b85a..f53a2f95 100644 --- a/rsconnect/bundle.py +++ b/rsconnect/bundle.py @@ -1102,6 +1102,27 @@ def validate_extra_files_(directory, extra_files): return result +def validate_extra_files(path, extra_files): + """ + If the path is a directory, validate that extra files all exist and are + beneath the given directory. + In the case that the path is a file, use the directory of the file for validation. + + :param path: a file or directory that the extra files must be relative to. + :param extra_files: the list of extra files to qualify and validate. + :return: the extra files qualified by the directory. + """ + base_dir = path + if isfile(path): + base_dir = dirname(path) + result = [] + for extra in extra_files: + if Path(extra).parent != Path(base_dir): + raise RSConnectException(f"{extra} must be under {base_dir}.") + result.append(extra) + return result + + def validate_manifest_file(file_or_directory): """ Validates that the name given represents either an existing manifest.json file or From ebcac592f0e285693754f7dc0b184f7776754562 Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Mon, 6 Feb 2023 19:44:17 -0500 Subject: [PATCH 42/97] fix create_file_list not excluding when path is a file --- rsconnect/bundle.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rsconnect/bundle.py b/rsconnect/bundle.py index f53a2f95..b9a4f152 100644 --- a/rsconnect/bundle.py +++ b/rsconnect/bundle.py @@ -786,15 +786,15 @@ def create_file_list( :param excludes: a sequence of glob patterns that will exclude matched files. :return: the list of relevant files, relative to the given directory. """ - extra_files = extra_files or [] - excludes = excludes if excludes else [] + extra_files = set(extra_files) if extra_files else set() + excludes = set(excludes) if excludes else set() glob_set = create_glob_set(path, excludes) exclude_paths = {Path(p) for p in excludes} file_set = set(extra_files) # type: typing.Set[str] if isfile(path): file_set.add(path) - return sorted(file_set) + return sorted(file_set - excludes) for subdir, dirs, files in os.walk(path): if Path(subdir) in exclude_paths: From 6a64af0bbade9a0ad08a5fc705ed602b09c710d0 Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Mon, 6 Feb 2023 19:45:09 -0500 Subject: [PATCH 43/97] check voila.json exists before including --- rsconnect/bundle.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/rsconnect/bundle.py b/rsconnect/bundle.py index b9a4f152..5a2f2468 100644 --- a/rsconnect/bundle.py +++ b/rsconnect/bundle.py @@ -890,15 +890,19 @@ def make_voila_bundle( :param image: the optional docker image to be specified for off-host execution. Default = None. :return: a file-like object containing the bundle tarball. """ - voila_config = "voila.json" extra_files = list(extra_files) if extra_files else [] - extra_files.append(voila_config) + + entrypoint = entrypoint or infer_entrypoint(path=path, mimetype="text/ipynb") + base_dir = dirname(entrypoint) + + voila_json_path = join(base_dir, "voila.json") + if os.path.isfile(voila_json_path): + extra_files.append(voila_json_path) manifest = create_voila_manifest(**locals()) if manifest.data.get("files") is None: return None - entrypoint = entrypoint or infer_entrypoint(path=path, mimetype="text/ipynb") - base_dir = dirname(entrypoint) + manifest_path = join(base_dir, "manifest.json") write_manifest_json(manifest_path, manifest.data) From a536e7039c6f2ae6b5444a276ff40d08d08a39e0 Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Mon, 6 Feb 2023 19:53:43 -0500 Subject: [PATCH 44/97] no longer auto prepend base_dir when adding files --- rsconnect/bundle.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/rsconnect/bundle.py b/rsconnect/bundle.py index 5a2f2468..afe5795b 100644 --- a/rsconnect/bundle.py +++ b/rsconnect/bundle.py @@ -1480,9 +1480,7 @@ def create_voila_manifest( excludes.extend(["manifest.json"]) file_list = create_file_list(path, extra_files, excludes) for rel_path in file_list: - path = join(base_dir, rel_path) if os.path.isdir(base_dir) else rel_path - manifest.add_file(path) - + manifest.add_file(rel_path) return manifest From 30584725a29e0a87fbd0705b8c96f3fe1194e34a Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Mon, 6 Feb 2023 19:58:43 -0500 Subject: [PATCH 45/97] Revert "fix create_file_list not excluding when path is a file" This reverts commit ebcac592f0e285693754f7dc0b184f7776754562. --- rsconnect/bundle.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rsconnect/bundle.py b/rsconnect/bundle.py index afe5795b..55ceafc8 100644 --- a/rsconnect/bundle.py +++ b/rsconnect/bundle.py @@ -786,15 +786,15 @@ def create_file_list( :param excludes: a sequence of glob patterns that will exclude matched files. :return: the list of relevant files, relative to the given directory. """ - extra_files = set(extra_files) if extra_files else set() - excludes = set(excludes) if excludes else set() + extra_files = extra_files or [] + excludes = excludes if excludes else [] glob_set = create_glob_set(path, excludes) exclude_paths = {Path(p) for p in excludes} file_set = set(extra_files) # type: typing.Set[str] if isfile(path): file_set.add(path) - return sorted(file_set - excludes) + return sorted(file_set) for subdir, dirs, files in os.walk(path): if Path(subdir) in exclude_paths: From 39a8765f41541d8b9cb3697b5c627f31d6f3de29 Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Mon, 6 Feb 2023 20:00:08 -0500 Subject: [PATCH 46/97] fix create_file_list not excluding when path is a file --- rsconnect/bundle.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rsconnect/bundle.py b/rsconnect/bundle.py index 55ceafc8..67f4cdc3 100644 --- a/rsconnect/bundle.py +++ b/rsconnect/bundle.py @@ -794,7 +794,7 @@ def create_file_list( if isfile(path): file_set.add(path) - return sorted(file_set) + return sorted(file_set - set(excludes)) for subdir, dirs, files in os.walk(path): if Path(subdir) in exclude_paths: From d434416b3533030b51169d048f8fabe7e18cc24c Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Mon, 6 Feb 2023 20:30:16 -0500 Subject: [PATCH 47/97] update validate_extra_files to include none case --- rsconnect/bundle.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/rsconnect/bundle.py b/rsconnect/bundle.py index 67f4cdc3..83419f2f 100644 --- a/rsconnect/bundle.py +++ b/rsconnect/bundle.py @@ -1116,14 +1116,13 @@ def validate_extra_files(path, extra_files): :param extra_files: the list of extra files to qualify and validate. :return: the extra files qualified by the directory. """ - base_dir = path - if isfile(path): - base_dir = dirname(path) result = [] - for extra in extra_files: - if Path(extra).parent != Path(base_dir): - raise RSConnectException(f"{extra} must be under {base_dir}.") - result.append(extra) + if extra_files: + base_dir = path if isdir(path) else dirname(path) + for extra in extra_files: + if Path(extra).parent != Path(base_dir): + raise RSConnectException(f"{extra} must be under {base_dir}.") + result.append(extra) return result From 5ee21897ee3e4bc8f47c06f7dc28fb098305e5dd Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Mon, 6 Feb 2023 20:31:19 -0500 Subject: [PATCH 48/97] update test_validate_extra_files --- tests/test_bundle.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_bundle.py b/tests/test_bundle.py index fce50af3..cfefbcef 100644 --- a/tests/test_bundle.py +++ b/tests/test_bundle.py @@ -612,7 +612,7 @@ def test_validate_extra_files(self): self.assertEqual(validate_extra_files(directory, []), []) self.assertEqual( validate_extra_files(directory, [join(directory, "index.htm")]), - ["index.htm"], + [os.path.join(directory, "index.htm")], ) def test_validate_title(self): From d76382c5f89d3f1d0b53ca3d8a4e38c336d21e70 Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Tue, 7 Feb 2023 13:42:11 -0500 Subject: [PATCH 49/97] Revert 5ee2189 to 2007e6d This reverts: 5ee2189 d434416 39a8765 3058472 a536e70 6a64af0 ebcac59 2220e98 2007e6d --- rsconnect/bundle.py | 40 +++++++++------------------------------- tests/test_bundle.py | 2 +- 2 files changed, 10 insertions(+), 32 deletions(-) diff --git a/rsconnect/bundle.py b/rsconnect/bundle.py index 83419f2f..8c779f1f 100644 --- a/rsconnect/bundle.py +++ b/rsconnect/bundle.py @@ -794,7 +794,7 @@ def create_file_list( if isfile(path): file_set.add(path) - return sorted(file_set - set(excludes)) + return sorted(file_set) for subdir, dirs, files in os.walk(path): if Path(subdir) in exclude_paths: @@ -890,19 +890,15 @@ def make_voila_bundle( :param image: the optional docker image to be specified for off-host execution. Default = None. :return: a file-like object containing the bundle tarball. """ + voila_config = "voila.json" extra_files = list(extra_files) if extra_files else [] - - entrypoint = entrypoint or infer_entrypoint(path=path, mimetype="text/ipynb") - base_dir = dirname(entrypoint) - - voila_json_path = join(base_dir, "voila.json") - if os.path.isfile(voila_json_path): - extra_files.append(voila_json_path) + extra_files.append(voila_config) manifest = create_voila_manifest(**locals()) if manifest.data.get("files") is None: return None - + entrypoint = entrypoint or infer_entrypoint(path=path, mimetype="text/ipynb") + base_dir = dirname(entrypoint) manifest_path = join(base_dir, "manifest.json") write_manifest_json(manifest_path, manifest.data) @@ -1082,7 +1078,7 @@ def validate_file_is_notebook(file_name): raise RSConnectException("A Jupyter notebook (.ipynb) file is required here.") -def validate_extra_files_(directory, extra_files): +def validate_extra_files(directory, extra_files): """ If the user specified a list of extra files, validate that they all exist and are beneath the given directory and, if so, return a list of them made relative to that @@ -1106,26 +1102,6 @@ def validate_extra_files_(directory, extra_files): return result -def validate_extra_files(path, extra_files): - """ - If the path is a directory, validate that extra files all exist and are - beneath the given directory. - In the case that the path is a file, use the directory of the file for validation. - - :param path: a file or directory that the extra files must be relative to. - :param extra_files: the list of extra files to qualify and validate. - :return: the extra files qualified by the directory. - """ - result = [] - if extra_files: - base_dir = path if isdir(path) else dirname(path) - for extra in extra_files: - if Path(extra).parent != Path(base_dir): - raise RSConnectException(f"{extra} must be under {base_dir}.") - result.append(extra) - return result - - def validate_manifest_file(file_or_directory): """ Validates that the name given represents either an existing manifest.json file or @@ -1479,7 +1455,9 @@ def create_voila_manifest( excludes.extend(["manifest.json"]) file_list = create_file_list(path, extra_files, excludes) for rel_path in file_list: - manifest.add_file(rel_path) + path = join(base_dir, rel_path) if os.path.isdir(base_dir) else rel_path + manifest.add_file(path) + return manifest diff --git a/tests/test_bundle.py b/tests/test_bundle.py index cfefbcef..fce50af3 100644 --- a/tests/test_bundle.py +++ b/tests/test_bundle.py @@ -612,7 +612,7 @@ def test_validate_extra_files(self): self.assertEqual(validate_extra_files(directory, []), []) self.assertEqual( validate_extra_files(directory, [join(directory, "index.htm")]), - [os.path.join(directory, "index.htm")], + ["index.htm"], ) def test_validate_title(self): From 5220175983755c98206f5e07b0476880f63deb61 Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Mon, 6 Feb 2023 19:45:09 -0500 Subject: [PATCH 50/97] check voila.json exists before including --- rsconnect/bundle.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/rsconnect/bundle.py b/rsconnect/bundle.py index 8c779f1f..c9c31e0b 100644 --- a/rsconnect/bundle.py +++ b/rsconnect/bundle.py @@ -890,15 +890,19 @@ def make_voila_bundle( :param image: the optional docker image to be specified for off-host execution. Default = None. :return: a file-like object containing the bundle tarball. """ - voila_config = "voila.json" extra_files = list(extra_files) if extra_files else [] - extra_files.append(voila_config) + + entrypoint = entrypoint or infer_entrypoint(path=path, mimetype="text/ipynb") + base_dir = dirname(entrypoint) + + voila_json_path = join(base_dir, "voila.json") + if os.path.isfile(voila_json_path): + extra_files.append(voila_json_path) manifest = create_voila_manifest(**locals()) if manifest.data.get("files") is None: return None - entrypoint = entrypoint or infer_entrypoint(path=path, mimetype="text/ipynb") - base_dir = dirname(entrypoint) + manifest_path = join(base_dir, "manifest.json") write_manifest_json(manifest_path, manifest.data) From 9c18efc86cea4c96bc2ec2f322c796f89ab7684f Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Tue, 7 Feb 2023 16:26:20 -0500 Subject: [PATCH 51/97] add manifest ability to change files relative to entrypoint - stage_to_deploy - make_relative_to_deploy_dir --- rsconnect/bundle.py | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/rsconnect/bundle.py b/rsconnect/bundle.py index c9c31e0b..26d4639d 100644 --- a/rsconnect/bundle.py +++ b/rsconnect/bundle.py @@ -16,6 +16,7 @@ from collections import defaultdict from mimetypes import guess_type from pathlib import Path +from copy import deepcopy import click @@ -139,18 +140,33 @@ def entrypoint(self, value): self.data["metadata"]["entrypoint"] = value def add_file(self, path): - base_dir = dirname(self.entrypoint) - path = join(base_dir, path) if os.path.isdir(self.entrypoint) else path self.data["files"][path] = {"checksum": file_checksum(path)} return self def discard_file(self, path): - base_dir = dirname(self.entrypoint) - path = join(base_dir, path) if os.path.isdir(self.entrypoint) else path if path in self.data["files"]: del self.data["files"][path] return self + def make_relative_to_deploy_dir(self): + rel_paths = {} + for path in self.data["files"]: + rel_path = os.path.relpath(path, dirname(self.entrypoint)) + rel_paths[rel_path] = self.data["files"][path] + self.data["files"] = rel_paths + self.entrypoint = os.path.relpath(self.entrypoint, dirname(self.entrypoint)) + return self + + def stage_to_deploy(self): + new_manifest = deepcopy(self) + new_data_files = {} + for path in self.data["files"]: + rel_path = os.path.relpath(path, dirname(self.entrypoint)) + new_data_files[rel_path] = self.data["files"][path] + new_manifest.data["files"] = new_data_files + new_manifest.entrypoint = os.path.relpath(self.entrypoint, dirname(self.entrypoint)) + return new_manifest + class Bundle: def __init__(self, *args, **kwargs) -> None: @@ -904,7 +920,7 @@ def make_voila_bundle( return None manifest_path = join(base_dir, "manifest.json") - write_manifest_json(manifest_path, manifest.data) + write_manifest_json(manifest_path, manifest.stage_to_deploy().data) bundle = Bundle() for f in manifest.data["files"]: From 69115aea4eabeaabc9fd9514649b5771d92e23da Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Tue, 7 Feb 2023 16:39:16 -0500 Subject: [PATCH 52/97] add Bundle ability to flatten to deploy_dir --- rsconnect/bundle.py | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/rsconnect/bundle.py b/rsconnect/bundle.py index 26d4639d..bfea7202 100644 --- a/rsconnect/bundle.py +++ b/rsconnect/bundle.py @@ -170,19 +170,29 @@ def stage_to_deploy(self): class Bundle: def __init__(self, *args, **kwargs) -> None: - self.file_locations: set = set() + self.file_paths: set = set() + self._deploy_dir = None + + @property + def deploy_dir(self): + return self._deploy_dir + + @deploy_dir.setter + def deploy_dir(self, value): + self._deploy_dir = value def add_file(self, filepath): - self.file_locations.add(filepath) + self.file_paths.add(filepath) def discard_file(self, filepath): - self.file_locations.discard(filepath) + self.file_paths.discard(filepath) - def to_file(self): + def to_file(self, flatten_to_deploy_dir=True): bundle_file = tempfile.TemporaryFile(prefix="rsc_bundle") with tarfile.open(mode="w:gz", fileobj=bundle_file) as bundle: - for file in self.file_locations: - bundle.add(file) + for fp in self.file_paths: + rel_path = Path(fp).relative_to(self.deploy_dir) if flatten_to_deploy_dir else None + bundle.add(fp, arcname=rel_path) bundle_file.seek(0) return bundle_file @@ -926,7 +936,7 @@ def make_voila_bundle( for f in manifest.data["files"]: bundle.add_file(f) bundle.add_file(manifest_path) - + bundle.deploy_dir = dirname(entrypoint) return bundle.to_file() From 4cf306bd6acf1080cf7129c674fb1676e35fa645 Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Tue, 7 Feb 2023 16:41:32 -0500 Subject: [PATCH 53/97] change if-expression to or --- rsconnect/bundle.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rsconnect/bundle.py b/rsconnect/bundle.py index bfea7202..74ff0071 100644 --- a/rsconnect/bundle.py +++ b/rsconnect/bundle.py @@ -1476,7 +1476,7 @@ def create_voila_manifest( validate_file_is_notebook(entrypoint) manifest.entrypoint = entrypoint else: - manifest.entrypoint = "" if not entrypoint else entrypoint # multi-notebook mode + manifest.entrypoint = entrypoint or "" # handle environment files if not exists(join(base_dir, environment.filename)) or force_generate: From e7738e16c3f31f7d0e3ee3cd7ef5ad042b48e3e4 Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Tue, 7 Feb 2023 19:22:29 -0500 Subject: [PATCH 54/97] store absolute path --- rsconnect/bundle.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/rsconnect/bundle.py b/rsconnect/bundle.py index 74ff0071..87d0be89 100644 --- a/rsconnect/bundle.py +++ b/rsconnect/bundle.py @@ -1485,9 +1485,8 @@ def create_voila_manifest( excludes.extend(["manifest.json"]) file_list = create_file_list(path, extra_files, excludes) for rel_path in file_list: - path = join(base_dir, rel_path) if os.path.isdir(base_dir) else rel_path - manifest.add_file(path) - + abs_path = join(base_dir, rel_path) + manifest.add_file(abs_path) return manifest From d0886e7654521078f93ddf46eacc4ecf2e6b702a Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Tue, 7 Feb 2023 19:23:57 -0500 Subject: [PATCH 55/97] rename base_dir to deploy_dir in create_voila_manifest --- rsconnect/bundle.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/rsconnect/bundle.py b/rsconnect/bundle.py index 87d0be89..c340a69a 100644 --- a/rsconnect/bundle.py +++ b/rsconnect/bundle.py @@ -1469,7 +1469,7 @@ def create_voila_manifest( excludes = list(excludes) if excludes else [] entrypoint = entrypoint or infer_entrypoint(path=path, mimetype="text/ipynb") - base_dir = dirname(entrypoint) + deploy_dir = dirname(entrypoint) manifest = Manifest(app_mode=AppModes.JUPYTER_VOILA, environment=environment, entrypoint=entrypoint, image=image) if isfile(path): @@ -1479,13 +1479,13 @@ def create_voila_manifest( manifest.entrypoint = entrypoint or "" # handle environment files - if not exists(join(base_dir, environment.filename)) or force_generate: - manifest.add_file(join(base_dir, environment.filename)) + if not exists(join(deploy_dir, environment.filename)) or force_generate: + manifest.add_file(join(deploy_dir, environment.filename)) excludes.extend(["manifest.json"]) file_list = create_file_list(path, extra_files, excludes) for rel_path in file_list: - abs_path = join(base_dir, rel_path) + abs_path = join(deploy_dir, rel_path) manifest.add_file(abs_path) return manifest From b2bae6a67c1f07aab3c113042873771efda0d9eb Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Thu, 9 Feb 2023 15:45:52 -0500 Subject: [PATCH 56/97] add buffer to Manifest --- rsconnect/bundle.py | 51 ++++++++++++++++++++++++++++++++------------- 1 file changed, 36 insertions(+), 15 deletions(-) diff --git a/rsconnect/bundle.py b/rsconnect/bundle.py index c340a69a..511d3e87 100644 --- a/rsconnect/bundle.py +++ b/rsconnect/bundle.py @@ -58,6 +58,7 @@ class Manifest: def __init__(self, *args, **kwargs) -> None: self.data = dict() + self.buffer = dict() version = kwargs.get("version") environment = kwargs.get("environment") app_mode = kwargs.get("app_mode") @@ -148,25 +149,48 @@ def discard_file(self, path): del self.data["files"][path] return self - def make_relative_to_deploy_dir(self): - rel_paths = {} - for path in self.data["files"]: - rel_path = os.path.relpath(path, dirname(self.entrypoint)) - rel_paths[rel_path] = self.data["files"][path] - self.data["files"] = rel_paths - self.entrypoint = os.path.relpath(self.entrypoint, dirname(self.entrypoint)) + def add_to_buffer(self, key, value): + self.buffer[key] = value + self.data["files"][key] = {"checksum": buffer_checksum(value)} return self - def stage_to_deploy(self): - new_manifest = deepcopy(self) + def discard_from_buffer(self, key): + if key in self.buffer: + del self.buffer[key] + del self.data["files"][key] + return self + + @property + def flattened_data(self): new_data_files = {} for path in self.data["files"]: rel_path = os.path.relpath(path, dirname(self.entrypoint)) new_data_files[rel_path] = self.data["files"][path] - new_manifest.data["files"] = new_data_files - new_manifest.entrypoint = os.path.relpath(self.entrypoint, dirname(self.entrypoint)) + return new_data_files + + @property + def flattened_buffer(self): + new_buffer = {} + for k, v in self.buffer.items(): + rel_path = os.path.relpath(k, dirname(self.entrypoint)) + new_buffer[rel_path] = v + return new_buffer + + @property + def flattened_entrypoint(self): + return os.path.relpath(self.entrypoint, dirname(self.entrypoint)) + + def stage_to_deploy(self): + new_manifest = deepcopy(self) + new_manifest.data["files"] = self.flattened_data + new_manifest.buffer = self.flattened_buffer + new_manifest.entrypoint = self.flattened_entrypoint return new_manifest + def make_relative_to_deploy_dir(self): + self = self.stage_to_deploy() + return self + class Bundle: def __init__(self, *args, **kwargs) -> None: @@ -1478,10 +1502,7 @@ def create_voila_manifest( else: manifest.entrypoint = entrypoint or "" - # handle environment files - if not exists(join(deploy_dir, environment.filename)) or force_generate: - manifest.add_file(join(deploy_dir, environment.filename)) - + manifest.add_to_buffer(join(deploy_dir, environment.filename), environment.contents) excludes.extend(["manifest.json"]) file_list = create_file_list(path, extra_files, excludes) for rel_path in file_list: From 9d29e9e355394cb6b6bc28c6db109deb45e04c75 Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Thu, 9 Feb 2023 15:51:39 -0500 Subject: [PATCH 57/97] rename base_dir to deploy_dir in make_voila_bundle --- rsconnect/bundle.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/rsconnect/bundle.py b/rsconnect/bundle.py index 511d3e87..db3ef566 100644 --- a/rsconnect/bundle.py +++ b/rsconnect/bundle.py @@ -943,9 +943,8 @@ def make_voila_bundle( extra_files = list(extra_files) if extra_files else [] entrypoint = entrypoint or infer_entrypoint(path=path, mimetype="text/ipynb") - base_dir = dirname(entrypoint) - - voila_json_path = join(base_dir, "voila.json") + deploy_dir = dirname(entrypoint) + voila_json_path = join(deploy_dir, "voila.json") if os.path.isfile(voila_json_path): extra_files.append(voila_json_path) @@ -953,7 +952,7 @@ def make_voila_bundle( if manifest.data.get("files") is None: return None - manifest_path = join(base_dir, "manifest.json") + manifest_path = join(deploy_dir, "manifest.json") write_manifest_json(manifest_path, manifest.stage_to_deploy().data) bundle = Bundle() From ffb730f670cd66c038a269556c54d34124d45654 Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Thu, 9 Feb 2023 15:52:47 -0500 Subject: [PATCH 58/97] add rscException for no valid manifest files --- rsconnect/bundle.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rsconnect/bundle.py b/rsconnect/bundle.py index db3ef566..671879e5 100644 --- a/rsconnect/bundle.py +++ b/rsconnect/bundle.py @@ -950,7 +950,7 @@ def make_voila_bundle( manifest = create_voila_manifest(**locals()) if manifest.data.get("files") is None: - return None + raise RSConnectException("No valid files were found for the manifest.") manifest_path = join(deploy_dir, "manifest.json") write_manifest_json(manifest_path, manifest.stage_to_deploy().data) From fcfbdd0bbc418a7a881378981a797c579a8cd88a Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Thu, 9 Feb 2023 15:55:28 -0500 Subject: [PATCH 59/97] add buffer to Bundle --- rsconnect/bundle.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/rsconnect/bundle.py b/rsconnect/bundle.py index 671879e5..0cbfca70 100644 --- a/rsconnect/bundle.py +++ b/rsconnect/bundle.py @@ -195,6 +195,7 @@ def make_relative_to_deploy_dir(self): class Bundle: def __init__(self, *args, **kwargs) -> None: self.file_paths: set = set() + self.buffer: dict = {} self._deploy_dir = None @property @@ -215,11 +216,27 @@ def to_file(self, flatten_to_deploy_dir=True): bundle_file = tempfile.TemporaryFile(prefix="rsc_bundle") with tarfile.open(mode="w:gz", fileobj=bundle_file) as bundle: for fp in self.file_paths: + if Path(fp).name in self.buffer: + continue rel_path = Path(fp).relative_to(self.deploy_dir) if flatten_to_deploy_dir else None bundle.add(fp, arcname=rel_path) + for k, v in self.buffer.items(): + buf = io.BytesIO(to_bytes(v)) + file_info = tarfile.TarInfo(k) + file_info.size = len(buf.getvalue()) + bundle.addfile(file_info, buf) bundle_file.seek(0) return bundle_file + def add_to_buffer(self, key, value): + self.buffer[key] = value + return self + + def discard_from_buffer(self, key): + if key in self.buffer: + del self.buffer[key] + return self + # noinspection SpellCheckingInspection def make_source_manifest( @@ -957,7 +974,11 @@ def make_voila_bundle( bundle = Bundle() for f in manifest.data["files"]: + if f in manifest.buffer: + continue bundle.add_file(f) + for k, v in manifest.flattened_buffer.items(): + bundle.add_to_buffer(k, v) bundle.add_file(manifest_path) bundle.deploy_dir = dirname(entrypoint) return bundle.to_file() From 5adda80255a593843771d0b4dd2ef3bc66165b4b Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Thu, 9 Feb 2023 15:59:22 -0500 Subject: [PATCH 60/97] add annotation per mypy --- rsconnect/bundle.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rsconnect/bundle.py b/rsconnect/bundle.py index 0cbfca70..f51533a9 100644 --- a/rsconnect/bundle.py +++ b/rsconnect/bundle.py @@ -58,7 +58,7 @@ class Manifest: def __init__(self, *args, **kwargs) -> None: self.data = dict() - self.buffer = dict() + self.buffer: dict = dict() version = kwargs.get("version") environment = kwargs.get("environment") app_mode = kwargs.get("app_mode") From d3ea774b2530ab39521d345a1e1d424ae438ff50 Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Thu, 9 Feb 2023 16:06:44 -0500 Subject: [PATCH 61/97] use stage_to_deploy for write_voila_manifest_json --- rsconnect/bundle.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rsconnect/bundle.py b/rsconnect/bundle.py index f51533a9..a9794347 100644 --- a/rsconnect/bundle.py +++ b/rsconnect/bundle.py @@ -1560,7 +1560,7 @@ def write_voila_manifest_json( entrypoint = entrypoint or infer_entrypoint(path=path, mimetype="text/ipynb") base_dir = dirname(entrypoint) manifest_path = join(base_dir, "manifest.json") - write_manifest_json(manifest_path, manifest.data) + write_manifest_json(manifest_path, manifest.stage_to_deploy().data) return exists(manifest_path) From 488a3791ba9bb0c8ea239a82b9a9dc4fed045fb9 Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Thu, 9 Feb 2023 17:20:45 -0500 Subject: [PATCH 62/97] replace os.path.relpath wtih relpath --- rsconnect/bundle.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/rsconnect/bundle.py b/rsconnect/bundle.py index a9794347..312c9a7b 100644 --- a/rsconnect/bundle.py +++ b/rsconnect/bundle.py @@ -164,7 +164,7 @@ def discard_from_buffer(self, key): def flattened_data(self): new_data_files = {} for path in self.data["files"]: - rel_path = os.path.relpath(path, dirname(self.entrypoint)) + rel_path = relpath(path, dirname(self.entrypoint)) new_data_files[rel_path] = self.data["files"][path] return new_data_files @@ -172,13 +172,13 @@ def flattened_data(self): def flattened_buffer(self): new_buffer = {} for k, v in self.buffer.items(): - rel_path = os.path.relpath(k, dirname(self.entrypoint)) + rel_path = relpath(k, dirname(self.entrypoint)) new_buffer[rel_path] = v return new_buffer @property def flattened_entrypoint(self): - return os.path.relpath(self.entrypoint, dirname(self.entrypoint)) + return relpath(self.entrypoint, dirname(self.entrypoint)) def stage_to_deploy(self): new_manifest = deepcopy(self) @@ -868,7 +868,7 @@ def create_file_list( continue for file in files: abs_path = os.path.join(subdir, file) - rel_path = os.path.relpath(abs_path, path) + rel_path = relpath(abs_path, path) if Path(abs_path) in exclude_paths: continue From cad1ae62a07b2b6326b79066bdf7d92044246c8a Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Thu, 9 Feb 2023 17:29:56 -0500 Subject: [PATCH 63/97] prioritize user provided requirements.txt --- rsconnect/bundle.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/rsconnect/bundle.py b/rsconnect/bundle.py index 312c9a7b..e238839a 100644 --- a/rsconnect/bundle.py +++ b/rsconnect/bundle.py @@ -1522,7 +1522,9 @@ def create_voila_manifest( else: manifest.entrypoint = entrypoint or "" - manifest.add_to_buffer(join(deploy_dir, environment.filename), environment.contents) + if not exists(join(deploy_dir, environment.filename)) or force_generate: + manifest.add_to_buffer(join(deploy_dir, environment.filename), environment.contents) + excludes.extend(["manifest.json"]) file_list = create_file_list(path, extra_files, excludes) for rel_path in file_list: From 20257a9b3b1a0bd693fd4e712bcc00bc1411de55 Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Fri, 10 Feb 2023 11:03:46 -0500 Subject: [PATCH 64/97] rename stage_to_deploy to flattened_copy --- rsconnect/bundle.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/rsconnect/bundle.py b/rsconnect/bundle.py index e238839a..2699b67a 100644 --- a/rsconnect/bundle.py +++ b/rsconnect/bundle.py @@ -180,7 +180,8 @@ def flattened_buffer(self): def flattened_entrypoint(self): return relpath(self.entrypoint, dirname(self.entrypoint)) - def stage_to_deploy(self): + @property + def flattened_copy(self): new_manifest = deepcopy(self) new_manifest.data["files"] = self.flattened_data new_manifest.buffer = self.flattened_buffer @@ -188,7 +189,7 @@ def stage_to_deploy(self): return new_manifest def make_relative_to_deploy_dir(self): - self = self.stage_to_deploy() + self = self.flattened_copy return self @@ -970,7 +971,7 @@ def make_voila_bundle( raise RSConnectException("No valid files were found for the manifest.") manifest_path = join(deploy_dir, "manifest.json") - write_manifest_json(manifest_path, manifest.stage_to_deploy().data) + write_manifest_json(manifest_path, manifest.flattened_copy.data) bundle = Bundle() for f in manifest.data["files"]: @@ -1562,7 +1563,7 @@ def write_voila_manifest_json( entrypoint = entrypoint or infer_entrypoint(path=path, mimetype="text/ipynb") base_dir = dirname(entrypoint) manifest_path = join(base_dir, "manifest.json") - write_manifest_json(manifest_path, manifest.stage_to_deploy().data) + write_manifest_json(manifest_path, manifest.flattened_copy.data) return exists(manifest_path) From 075b84a402e7bccc5cc6074259e8eeb714d115a9 Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Fri, 10 Feb 2023 13:16:32 -0500 Subject: [PATCH 65/97] use create_python_environment --- rsconnect/main.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/rsconnect/main.py b/rsconnect/main.py index 73baecfd..5c5dd001 100644 --- a/rsconnect/main.py +++ b/rsconnect/main.py @@ -903,15 +903,12 @@ def deploy_voila( kwargs = locals() set_verbosity(verbose) app_mode = AppModes.JUPYTER_VOILA - python, environment = get_python_env_info(path, python, conda_mode=False, force_generate=force_generate) kwargs["extra_files"] = extra_files = validate_extra_files(dirname(path), extra_files) - base_dir = dirname(path) - _warn_on_ignored_manifest(base_dir) - _warn_if_no_requirements_file(base_dir) - _warn_if_environment_directory(base_dir) - if force_generate: - _warn_on_ignored_requirements(base_dir, environment.filename) - + environment = create_python_environment( + path if isdir(path) else dirname(path), + force_generate, + python, + ) ce = RSConnectExecutor(**kwargs).validate_server().validate_app_mode(app_mode=app_mode) ce.make_bundle( make_voila_bundle, From dfc9121760d3869d7a413c897c204a50876dd498 Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Fri, 10 Feb 2023 13:45:00 -0500 Subject: [PATCH 66/97] return just the file name when path is a file existing behavior adds deploy directory as prefix --- rsconnect/bundle.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rsconnect/bundle.py b/rsconnect/bundle.py index 2699b67a..0075c496 100644 --- a/rsconnect/bundle.py +++ b/rsconnect/bundle.py @@ -861,7 +861,7 @@ def create_file_list( file_set = set(extra_files) # type: typing.Set[str] if isfile(path): - file_set.add(path) + file_set.add(Path(path).name) return sorted(file_set) for subdir, dirs, files in os.walk(path): From dba24ee41d01ef18450e6401d40da8a24d51dfa2 Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Fri, 10 Feb 2023 15:41:37 -0500 Subject: [PATCH 67/97] add_relative_path --- rsconnect/bundle.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/rsconnect/bundle.py b/rsconnect/bundle.py index 0075c496..80fe9e15 100644 --- a/rsconnect/bundle.py +++ b/rsconnect/bundle.py @@ -144,6 +144,14 @@ def add_file(self, path): self.data["files"][path] = {"checksum": file_checksum(path)} return self + def add_relative_path(self, path): + """ + Assumes that path resides below the deployment directory, construct that path add it to the manifest. + """ + mod_path = join(dirname(self.entrypoint), path) + self.data["files"][mod_path] = {"checksum": file_checksum(mod_path)} + return self + def discard_file(self, path): if path in self.data["files"]: del self.data["files"][path] @@ -1529,8 +1537,7 @@ def create_voila_manifest( excludes.extend(["manifest.json"]) file_list = create_file_list(path, extra_files, excludes) for rel_path in file_list: - abs_path = join(deploy_dir, rel_path) - manifest.add_file(abs_path) + manifest.add_relative_path(rel_path) return manifest From cfb667850efe8dc073ad041ade101476e2728219 Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Fri, 10 Feb 2023 17:06:48 -0500 Subject: [PATCH 68/97] exclude environment files, rely on Environment --- rsconnect/bundle.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rsconnect/bundle.py b/rsconnect/bundle.py index 80fe9e15..05f87585 100644 --- a/rsconnect/bundle.py +++ b/rsconnect/bundle.py @@ -1520,6 +1520,8 @@ def create_voila_manifest( """ extra_files = list(extra_files) if extra_files else [] excludes = list(excludes) if excludes else [] + excludes.extend([environment.filename, "manifest.json"]) + excludes.extend(list_environment_dirs(path)) entrypoint = entrypoint or infer_entrypoint(path=path, mimetype="text/ipynb") deploy_dir = dirname(entrypoint) @@ -1531,10 +1533,8 @@ def create_voila_manifest( else: manifest.entrypoint = entrypoint or "" - if not exists(join(deploy_dir, environment.filename)) or force_generate: - manifest.add_to_buffer(join(deploy_dir, environment.filename), environment.contents) + manifest.add_to_buffer(join(deploy_dir, environment.filename), environment.contents) - excludes.extend(["manifest.json"]) file_list = create_file_list(path, extra_files, excludes) for rel_path in file_list: manifest.add_relative_path(rel_path) From 355b2c14da1f2cfd8dce7f93c1bd415ef9e6cdf1 Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Tue, 14 Feb 2023 11:05:22 -0500 Subject: [PATCH 69/97] use directory for list_environment_dirs --- rsconnect/bundle.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rsconnect/bundle.py b/rsconnect/bundle.py index 05f87585..551031ba 100644 --- a/rsconnect/bundle.py +++ b/rsconnect/bundle.py @@ -1521,10 +1521,10 @@ def create_voila_manifest( extra_files = list(extra_files) if extra_files else [] excludes = list(excludes) if excludes else [] excludes.extend([environment.filename, "manifest.json"]) - excludes.extend(list_environment_dirs(path)) entrypoint = entrypoint or infer_entrypoint(path=path, mimetype="text/ipynb") deploy_dir = dirname(entrypoint) + excludes.extend(list_environment_dirs(deploy_dir)) manifest = Manifest(app_mode=AppModes.JUPYTER_VOILA, environment=environment, entrypoint=entrypoint, image=image) if isfile(path): From b3bcef63ba4859c680acef62503e2f80f0816912 Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Tue, 14 Feb 2023 11:09:34 -0500 Subject: [PATCH 70/97] use abspath on entrypoint --- rsconnect/bundle.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rsconnect/bundle.py b/rsconnect/bundle.py index 551031ba..995b2226 100644 --- a/rsconnect/bundle.py +++ b/rsconnect/bundle.py @@ -968,7 +968,7 @@ def make_voila_bundle( """ extra_files = list(extra_files) if extra_files else [] - entrypoint = entrypoint or infer_entrypoint(path=path, mimetype="text/ipynb") + entrypoint = entrypoint or infer_entrypoint(path=abspath(path), mimetype="text/ipynb") deploy_dir = dirname(entrypoint) voila_json_path = join(deploy_dir, "voila.json") if os.path.isfile(voila_json_path): @@ -1522,7 +1522,7 @@ def create_voila_manifest( excludes = list(excludes) if excludes else [] excludes.extend([environment.filename, "manifest.json"]) - entrypoint = entrypoint or infer_entrypoint(path=path, mimetype="text/ipynb") + entrypoint = entrypoint or infer_entrypoint(path=abspath(path), mimetype="text/ipynb") deploy_dir = dirname(entrypoint) excludes.extend(list_environment_dirs(deploy_dir)) manifest = Manifest(app_mode=AppModes.JUPYTER_VOILA, environment=environment, entrypoint=entrypoint, image=image) From ce06ff63bb7535a8dfeebc975601378240b077e9 Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Tue, 14 Feb 2023 11:58:41 -0500 Subject: [PATCH 71/97] add exclude to deploy_voila --- rsconnect/main.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/rsconnect/main.py b/rsconnect/main.py index 5c5dd001..63859396 100644 --- a/rsconnect/main.py +++ b/rsconnect/main.py @@ -852,6 +852,16 @@ def deploy_notebook( ) @server_args @content_args +@click.option( + "--exclude", + "-x", + multiple=True, + help=( + "Specify a glob pattern for ignoring files when building the bundle. Note that your shell may try " + "to expand this which will not do what you expect. Generally, it's safest to quote the pattern. " + "This option may be repeated." + ), +) @click.option( "--python", "-p", @@ -886,7 +896,7 @@ def deploy_voila( python=None, force_generate=False, extra_files=None, - excludes=None, + exclude=None, image: str = "", title: str = None, env_vars: typing.Dict[str, str] = None, @@ -915,7 +925,7 @@ def deploy_voila( path, entrypoint, extra_files, - excludes, + exclude, force_generate, environment, image=image, From 69cf3f0cd7b0839d04abad8aab669a05cac2bc13 Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Tue, 14 Feb 2023 14:36:58 -0500 Subject: [PATCH 72/97] check parent dir for exclude --- rsconnect/bundle.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/rsconnect/bundle.py b/rsconnect/bundle.py index 995b2226..e56885e5 100644 --- a/rsconnect/bundle.py +++ b/rsconnect/bundle.py @@ -872,11 +872,13 @@ def create_file_list( file_set.add(Path(path).name) return sorted(file_set) - for subdir, dirs, files in os.walk(path): - if Path(subdir) in exclude_paths: + for cur_dir, sub_dirs, files in os.walk(path): + if Path(cur_dir) in exclude_paths: + continue + if any(parent in exclude_paths for parent in Path(cur_dir).parents): continue for file in files: - abs_path = os.path.join(subdir, file) + abs_path = os.path.join(cur_dir, file) rel_path = relpath(abs_path, path) if Path(abs_path) in exclude_paths: From 8675207a3bdb7eedae803024f2339418d0c2800c Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Wed, 15 Feb 2023 18:33:54 -0500 Subject: [PATCH 73/97] split candidates from infer_entrypoint --- rsconnect/bundle.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/rsconnect/bundle.py b/rsconnect/bundle.py index e56885e5..5df4f620 100644 --- a/rsconnect/bundle.py +++ b/rsconnect/bundle.py @@ -891,10 +891,15 @@ def create_file_list( def infer_entrypoint(path, mimetype): - if os.path.isfile(path): - return path - if not os.path.isdir(path): - raise ValueError("Entrypoint is not a valid file type or directory.") + candidates = infer_entrypoint_candidates(path, mimetype) + return candidates.pop() if len(candidates) == 1 else None + + +def infer_entrypoint_candidates(path, mimetype) -> List: + if isfile(path): + return [path] + if not isdir(path): + raise RSConnectException("Entrypoint is not a valid file type or directory.") default_mimetype_entrypoints = defaultdict(str) default_mimetype_entrypoints["text/html"] = "index.html" @@ -903,13 +908,12 @@ def infer_entrypoint(path, mimetype): for file in os.listdir(path): rel_path = os.path.join(path, file) - if not os.path.isfile(rel_path): + if not isfile(rel_path): continue mimetype_filelist[guess_type(file)[0]].append(rel_path) if file in default_mimetype_entrypoints[mimetype]: return file - res = mimetype_filelist[mimetype].pop() if len(mimetype_filelist[mimetype]) == 1 else None - return res + return mimetype_filelist[mimetype] or [] def make_html_bundle( From 968e5352f12be60847de29215d2018d0f7f68db9 Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Thu, 16 Feb 2023 07:45:42 -0500 Subject: [PATCH 74/97] fix lack of CLI entrypoint for deploy_voila --- rsconnect/main.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/rsconnect/main.py b/rsconnect/main.py index 63859396..b9bbe229 100644 --- a/rsconnect/main.py +++ b/rsconnect/main.py @@ -852,6 +852,11 @@ def deploy_notebook( ) @server_args @content_args +@click.option( + "--entrypoint", + "-e", + help=("The module and executable object which serves as the entry point."), +) @click.option( "--exclude", "-x", From cdd27b1b9964d87d0aa8b31fd037284e7cb4be04 Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Thu, 16 Feb 2023 08:36:32 -0500 Subject: [PATCH 75/97] manifest rewrite for multi-notebook mode --- rsconnect/bundle.py | 80 +++++++++++++++++++++++++++++++++++++-------- 1 file changed, 67 insertions(+), 13 deletions(-) diff --git a/rsconnect/bundle.py b/rsconnect/bundle.py index 5df4f620..ce220800 100644 --- a/rsconnect/bundle.py +++ b/rsconnect/bundle.py @@ -17,6 +17,7 @@ from mimetypes import guess_type from pathlib import Path from copy import deepcopy +from typing import List import click @@ -148,7 +149,8 @@ def add_relative_path(self, path): """ Assumes that path resides below the deployment directory, construct that path add it to the manifest. """ - mod_path = join(dirname(self.entrypoint), path) + deploy_dir = dirname(self.entrypoint) if isfile(self.entrypoint) else self.entrypoint + mod_path = join(deploy_dir, path) self.data["files"][mod_path] = {"checksum": file_checksum(mod_path)} return self @@ -171,16 +173,18 @@ def discard_from_buffer(self, key): @property def flattened_data(self): new_data_files = {} + deploy_dir = dirname(self.entrypoint) if isfile(self.entrypoint) else self.entrypoint for path in self.data["files"]: - rel_path = relpath(path, dirname(self.entrypoint)) + rel_path = relpath(path, deploy_dir) new_data_files[rel_path] = self.data["files"][path] return new_data_files @property def flattened_buffer(self): new_buffer = {} + deploy_dir = dirname(self.entrypoint) if isfile(self.entrypoint) else self.entrypoint for k, v in self.buffer.items(): - rel_path = relpath(k, dirname(self.entrypoint)) + rel_path = relpath(k, deploy_dir) new_buffer[rel_path] = v return new_buffer @@ -973,11 +977,35 @@ def make_voila_bundle( :return: a file-like object containing the bundle tarball. """ extra_files = list(extra_files) if extra_files else [] + entrypoint_candidates = infer_entrypoint_candidates(path=abspath(path), mimetype="text/ipynb") + multi_notebook_mode = False + deploy_dir = abspath(path) + + if len(entrypoint_candidates) <= 0: + if entrypoint is None: + raise RSConnectException("No valid entrypoint found.") + deploy_dir = dirname(entrypoint) + elif len(entrypoint_candidates) == 1: + entrypoint = entrypoint or entrypoint_candidates[0] + deploy_dir = dirname(entrypoint) + else: # len(entrypoint_candidates) > 1: + if entrypoint is None: + raise RSConnectException( + """ + Unable to infer entrypoint from multiple candidates. + Multi-notebook deployments need to be specified with the following: + 1) An empty string entrypoint, i.e. "" + 2) A directory as the path + """ + ) + elif entrypoint == "": + multi_notebook_mode = True + deploy_dir = entrypoint = abspath(path) + else: + deploy_dir = dirname(entrypoint) - entrypoint = entrypoint or infer_entrypoint(path=abspath(path), mimetype="text/ipynb") - deploy_dir = dirname(entrypoint) voila_json_path = join(deploy_dir, "voila.json") - if os.path.isfile(voila_json_path): + if isfile(voila_json_path): extra_files.append(voila_json_path) manifest = create_voila_manifest(**locals()) @@ -985,7 +1013,10 @@ def make_voila_bundle( raise RSConnectException("No valid files were found for the manifest.") manifest_path = join(deploy_dir, "manifest.json") - write_manifest_json(manifest_path, manifest.flattened_copy.data) + manifest_flattened_copy_data = manifest.flattened_copy.data + if multi_notebook_mode and "metadata" in manifest_flattened_copy_data: + manifest_flattened_copy_data["metadata"]["entrypoint"] = "" + write_manifest_json(manifest_path, manifest_flattened_copy_data) bundle = Bundle() for f in manifest.data["files"]: @@ -995,7 +1026,7 @@ def make_voila_bundle( for k, v in manifest.flattened_buffer.items(): bundle.add_to_buffer(k, v) bundle.add_file(manifest_path) - bundle.deploy_dir = dirname(entrypoint) + bundle.deploy_dir = deploy_dir return bundle.to_file() @@ -1527,9 +1558,33 @@ def create_voila_manifest( extra_files = list(extra_files) if extra_files else [] excludes = list(excludes) if excludes else [] excludes.extend([environment.filename, "manifest.json"]) + deploy_dir = kwargs.get("deploy_dir") + + if not deploy_dir: + entrypoint_candidates = infer_entrypoint_candidates(path=abspath(path), mimetype="text/ipynb") + deploy_dir = abspath(path) + if len(entrypoint_candidates) <= 0: + if entrypoint is None: + raise RSConnectException("No valid entrypoint found.") + deploy_dir = dirname(entrypoint) + elif len(entrypoint_candidates) == 1: + entrypoint = entrypoint or entrypoint_candidates[0] + deploy_dir = dirname(entrypoint) + else: # len(entrypoint_candidates) > 1: + if entrypoint is None: + raise RSConnectException( + """ + Unable to infer entrypoint from multiple candidates. + Multi-notebook deployments need to be specified with the following: + 1) An empty string entrypoint, i.e. "" + 2) A directory as the path + """ + ) + elif entrypoint == "": + deploy_dir = entrypoint = abspath(path) + else: + deploy_dir = dirname(entrypoint) - entrypoint = entrypoint or infer_entrypoint(path=abspath(path), mimetype="text/ipynb") - deploy_dir = dirname(entrypoint) excludes.extend(list_environment_dirs(deploy_dir)) manifest = Manifest(app_mode=AppModes.JUPYTER_VOILA, environment=environment, entrypoint=entrypoint, image=image) @@ -1573,9 +1628,8 @@ def write_voila_manifest_json( :return: whether the manifest was written. """ manifest = create_voila_manifest(**locals()) - entrypoint = entrypoint or infer_entrypoint(path=path, mimetype="text/ipynb") - base_dir = dirname(entrypoint) - manifest_path = join(base_dir, "manifest.json") + deploy_dir = dirname(manifest.entrypoint) if isfile(manifest.entrypoint) else manifest.entrypoint + manifest_path = join(deploy_dir, "manifest.json") write_manifest_json(manifest_path, manifest.flattened_copy.data) return exists(manifest_path) From a20c89f5cbdf8d4def4cb7e67197788820864c79 Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Thu, 16 Feb 2023 09:46:36 -0500 Subject: [PATCH 76/97] set json dumps indent --- rsconnect/bundle.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rsconnect/bundle.py b/rsconnect/bundle.py index ce220800..faba2aae 100644 --- a/rsconnect/bundle.py +++ b/rsconnect/bundle.py @@ -127,7 +127,7 @@ def from_json_file(cls, json_path): @property def json(self): - return json.dumps(self.data) + return json.dumps(self.data, indent=2) @property def entrypoint(self): @@ -618,7 +618,7 @@ def make_notebook_html_bundle( # manifest manifest = make_html_manifest(filename, image) - bundle_add_buffer(bundle, "manifest.json", json.dumps(manifest)) + bundle_add_buffer(bundle, "manifest.json", json.dumps(manifest, indent=2)) # rewind file pointer bundle_file.seek(0) From 6ba4a382f6bbe5ea6cc32593e0ee5f026884c86e Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Thu, 16 Feb 2023 09:47:04 -0500 Subject: [PATCH 77/97] do not write manifest.json when bundling --- rsconnect/bundle.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/rsconnect/bundle.py b/rsconnect/bundle.py index faba2aae..59b3c9aa 100644 --- a/rsconnect/bundle.py +++ b/rsconnect/bundle.py @@ -1012,12 +1012,6 @@ def make_voila_bundle( if manifest.data.get("files") is None: raise RSConnectException("No valid files were found for the manifest.") - manifest_path = join(deploy_dir, "manifest.json") - manifest_flattened_copy_data = manifest.flattened_copy.data - if multi_notebook_mode and "metadata" in manifest_flattened_copy_data: - manifest_flattened_copy_data["metadata"]["entrypoint"] = "" - write_manifest_json(manifest_path, manifest_flattened_copy_data) - bundle = Bundle() for f in manifest.data["files"]: if f in manifest.buffer: @@ -1025,8 +1019,13 @@ def make_voila_bundle( bundle.add_file(f) for k, v in manifest.flattened_buffer.items(): bundle.add_to_buffer(k, v) - bundle.add_file(manifest_path) + + manifest_flattened_copy_data = manifest.flattened_copy.data + if multi_notebook_mode and "metadata" in manifest_flattened_copy_data: + manifest_flattened_copy_data["metadata"]["entrypoint"] = "" + bundle.add_to_buffer("manifest.json", json.dumps(manifest_flattened_copy_data, indent=2)) bundle.deploy_dir = deploy_dir + return bundle.to_file() From e507f104af8e5736994020c3d9231410bb7e0092 Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Thu, 16 Feb 2023 11:39:57 -0500 Subject: [PATCH 78/97] validate_extra_files when directly writing manifest --- rsconnect/bundle.py | 1 + 1 file changed, 1 insertion(+) diff --git a/rsconnect/bundle.py b/rsconnect/bundle.py index 59b3c9aa..66b87dbd 100644 --- a/rsconnect/bundle.py +++ b/rsconnect/bundle.py @@ -1584,6 +1584,7 @@ def create_voila_manifest( else: deploy_dir = dirname(entrypoint) + extra_files = validate_extra_files(deploy_dir, extra_files) excludes.extend(list_environment_dirs(deploy_dir)) manifest = Manifest(app_mode=AppModes.JUPYTER_VOILA, environment=environment, entrypoint=entrypoint, image=image) From 0a3c5db62ea0484d72afbe7a21f9a2980adf9856 Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Thu, 16 Feb 2023 16:59:02 -0500 Subject: [PATCH 79/97] add guess_deploy_dir & abs_entrypoint --- rsconnect/bundle.py | 68 +++++++++++++++++++++++++-------------------- 1 file changed, 38 insertions(+), 30 deletions(-) diff --git a/rsconnect/bundle.py b/rsconnect/bundle.py index 66b87dbd..c31ad25f 100644 --- a/rsconnect/bundle.py +++ b/rsconnect/bundle.py @@ -952,6 +952,37 @@ def make_html_bundle( return bundle_file +def guess_deploy_dir(path, entrypoint): + deploy_dir = None + if isfile(path): + if not entrypoint: + deploy_dir = dirname(abspath(path)) + if isfile(entrypoint) and path != entrypoint: + raise RSConnectException("Path and entrypoint need to match if they are are files.") + if isdir(entrypoint): + raise RSConnectException("Entrypoint cannot be a directory while the path is a file.") + elif isdir(path): + if not entrypoint: + deploy_dir = abspath(path) + elif isdir(entrypoint): + raise RSConnectException("Path and entrypoint cannot both be directories.") + guess_entry_file = os.path.join(abspath(path), basename(entrypoint)) + if isfile(guess_entry_file): + deploy_dir = dirname(guess_entry_file) + else: + deploy_dir = abspath(path) + return deploy_dir + + +def abs_entrypoint(path, entrypoint): + if isfile(entrypoint): + return abspath(entrypoint) + guess_entry_file = os.path.join(abspath(path), basename(entrypoint)) + if isfile(guess_entry_file): + return guess_entry_file + return None + + def make_voila_bundle( path: str, entrypoint: str, @@ -979,15 +1010,15 @@ def make_voila_bundle( extra_files = list(extra_files) if extra_files else [] entrypoint_candidates = infer_entrypoint_candidates(path=abspath(path), mimetype="text/ipynb") multi_notebook_mode = False - deploy_dir = abspath(path) if len(entrypoint_candidates) <= 0: if entrypoint is None: raise RSConnectException("No valid entrypoint found.") - deploy_dir = dirname(entrypoint) elif len(entrypoint_candidates) == 1: - entrypoint = entrypoint or entrypoint_candidates[0] - deploy_dir = dirname(entrypoint) + if entrypoint: + entrypoint = abs_entrypoint(path, entrypoint) + else: + entrypoint = entrypoint_candidates[0] else: # len(entrypoint_candidates) > 1: if entrypoint is None: raise RSConnectException( @@ -1000,10 +1031,9 @@ def make_voila_bundle( ) elif entrypoint == "": multi_notebook_mode = True - deploy_dir = entrypoint = abspath(path) - else: - deploy_dir = dirname(entrypoint) + entrypoint = abspath(path) + deploy_dir = guess_deploy_dir(path, entrypoint) voila_json_path = join(deploy_dir, "voila.json") if isfile(voila_json_path): extra_files.append(voila_json_path) @@ -1560,29 +1590,7 @@ def create_voila_manifest( deploy_dir = kwargs.get("deploy_dir") if not deploy_dir: - entrypoint_candidates = infer_entrypoint_candidates(path=abspath(path), mimetype="text/ipynb") - deploy_dir = abspath(path) - if len(entrypoint_candidates) <= 0: - if entrypoint is None: - raise RSConnectException("No valid entrypoint found.") - deploy_dir = dirname(entrypoint) - elif len(entrypoint_candidates) == 1: - entrypoint = entrypoint or entrypoint_candidates[0] - deploy_dir = dirname(entrypoint) - else: # len(entrypoint_candidates) > 1: - if entrypoint is None: - raise RSConnectException( - """ - Unable to infer entrypoint from multiple candidates. - Multi-notebook deployments need to be specified with the following: - 1) An empty string entrypoint, i.e. "" - 2) A directory as the path - """ - ) - elif entrypoint == "": - deploy_dir = entrypoint = abspath(path) - else: - deploy_dir = dirname(entrypoint) + deploy_dir = guess_deploy_dir(path, entrypoint) extra_files = validate_extra_files(deploy_dir, extra_files) excludes.extend(list_environment_dirs(deploy_dir)) From 90ce28658961337c31958b890916a2a60085d8d5 Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Thu, 16 Feb 2023 17:24:54 -0500 Subject: [PATCH 80/97] add --multi-notebook flag change multi-notebook deployment from an empty entrypoint string to the -m flag --- rsconnect/bundle.py | 32 +++++++++++++++++++------------- rsconnect/main.py | 16 ++++++++++++++++ 2 files changed, 35 insertions(+), 13 deletions(-) diff --git a/rsconnect/bundle.py b/rsconnect/bundle.py index c31ad25f..f4aa3083 100644 --- a/rsconnect/bundle.py +++ b/rsconnect/bundle.py @@ -966,9 +966,10 @@ def guess_deploy_dir(path, entrypoint): deploy_dir = abspath(path) elif isdir(entrypoint): raise RSConnectException("Path and entrypoint cannot both be directories.") - guess_entry_file = os.path.join(abspath(path), basename(entrypoint)) - if isfile(guess_entry_file): - deploy_dir = dirname(guess_entry_file) + elif entrypoint: + guess_entry_file = os.path.join(abspath(path), basename(entrypoint)) + if isfile(guess_entry_file): + deploy_dir = dirname(guess_entry_file) else: deploy_dir = abspath(path) return deploy_dir @@ -991,6 +992,7 @@ def make_voila_bundle( force_generate: bool, environment: Environment, image: str = None, + multi_notebook: bool = False, ) -> typing.IO[bytes]: """ Create an voila bundle, given a path and/or entrypoint. @@ -1009,7 +1011,6 @@ def make_voila_bundle( """ extra_files = list(extra_files) if extra_files else [] entrypoint_candidates = infer_entrypoint_candidates(path=abspath(path), mimetype="text/ipynb") - multi_notebook_mode = False if len(entrypoint_candidates) <= 0: if entrypoint is None: @@ -1020,20 +1021,20 @@ def make_voila_bundle( else: entrypoint = entrypoint_candidates[0] else: # len(entrypoint_candidates) > 1: - if entrypoint is None: + if entrypoint is None and not multi_notebook: raise RSConnectException( """ Unable to infer entrypoint from multiple candidates. - Multi-notebook deployments need to be specified with the following: - 1) An empty string entrypoint, i.e. "" - 2) A directory as the path + Multi-notebook deployments need to be specified with the following: + 1) A directory as the path + 2) Set multi_notebook=True, + i.e. --multi-notebook """ ) - elif entrypoint == "": - multi_notebook_mode = True - entrypoint = abspath(path) deploy_dir = guess_deploy_dir(path, entrypoint) + if multi_notebook: + deploy_dir = entrypoint = abspath(path) voila_json_path = join(deploy_dir, "voila.json") if isfile(voila_json_path): extra_files.append(voila_json_path) @@ -1051,7 +1052,7 @@ def make_voila_bundle( bundle.add_to_buffer(k, v) manifest_flattened_copy_data = manifest.flattened_copy.data - if multi_notebook_mode and "metadata" in manifest_flattened_copy_data: + if multi_notebook and "metadata" in manifest_flattened_copy_data: manifest_flattened_copy_data["metadata"]["entrypoint"] = "" bundle.add_to_buffer("manifest.json", json.dumps(manifest_flattened_copy_data, indent=2)) bundle.deploy_dir = deploy_dir @@ -1567,6 +1568,7 @@ def create_voila_manifest( excludes: typing.List[str] = None, force_generate: bool = True, image: str = None, + multi_notebook: bool = False, **kwargs ) -> Manifest: """ @@ -1619,6 +1621,7 @@ def write_voila_manifest_json( excludes: typing.List[str] = None, force_generate: bool = True, image: str = None, + multi_notebook: bool = False, ) -> bool: """ Creates and writes a manifest.json file for the given path. @@ -1637,8 +1640,11 @@ def write_voila_manifest_json( """ manifest = create_voila_manifest(**locals()) deploy_dir = dirname(manifest.entrypoint) if isfile(manifest.entrypoint) else manifest.entrypoint + manifest_flattened_copy_data = manifest.flattened_copy.data + if multi_notebook and "metadata" in manifest_flattened_copy_data: + manifest_flattened_copy_data["metadata"]["entrypoint"] = "" manifest_path = join(deploy_dir, "manifest.json") - write_manifest_json(manifest_path, manifest.flattened_copy.data) + write_manifest_json(manifest_path, manifest_flattened_copy_data) return exists(manifest_path) diff --git a/rsconnect/main.py b/rsconnect/main.py index b9bbe229..c90dbfe1 100644 --- a/rsconnect/main.py +++ b/rsconnect/main.py @@ -857,6 +857,12 @@ def deploy_notebook( "-e", help=("The module and executable object which serves as the entry point."), ) +@click.option( + "--multi-notebook", + "-m", + is_flag=True, + help=("Deploy in multi-notebook mode."), +) @click.option( "--exclude", "-x", @@ -914,6 +920,7 @@ def deploy_voila( insecure: bool = False, cacert: typing.IO = None, connect_server: api.RSConnectServer = None, + multi_notebook: bool = False, ): kwargs = locals() set_verbosity(verbose) @@ -934,6 +941,7 @@ def deploy_voila( force_generate, environment, image=image, + multi_notebook=multi_notebook, ).deploy_bundle().save_deployed_info().emit_task_log() @@ -1500,6 +1508,12 @@ def write_manifest_notebook( "This option may be repeated." ), ) +@click.option( + "--multi-notebook", + "-m", + is_flag=True, + help=("Set the manifest for multi-notebook mode."), +) def write_manifest_voila( path: str, entrypoint: str, @@ -1510,6 +1524,7 @@ def write_manifest_voila( extra_files, exclude, image, + multi_notebook, ): set_verbosity(verbose) with cli_feedback("Checking arguments"): @@ -1546,6 +1561,7 @@ def write_manifest_voila( exclude, force_generate, image, + multi_notebook, ) From 5d72f6f78a8a67fbb36585407a1391e3b0e28c88 Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Thu, 16 Feb 2023 18:07:56 -0500 Subject: [PATCH 81/97] Move validation to create_voila_manifest Since it is at the crossroads of deploy and write-manifest. --- rsconnect/bundle.py | 81 +++++++++++++++++++++++---------------------- 1 file changed, 42 insertions(+), 39 deletions(-) diff --git a/rsconnect/bundle.py b/rsconnect/bundle.py index f4aa3083..3dd6d22e 100644 --- a/rsconnect/bundle.py +++ b/rsconnect/bundle.py @@ -60,6 +60,7 @@ class Manifest: def __init__(self, *args, **kwargs) -> None: self.data = dict() self.buffer: dict = dict() + self._deploy_dir = None version = kwargs.get("version") environment = kwargs.get("environment") app_mode = kwargs.get("app_mode") @@ -116,6 +117,14 @@ def __init__(self, *args, **kwargs) -> None: self.data["files"] = {} + @property + def deploy_dir(self): + return self._deploy_dir + + @deploy_dir.setter + def deploy_dir(self, value): + self._deploy_dir = value + @classmethod def from_json(cls, json_str): return cls(json.loads(json_str)) @@ -1009,35 +1018,6 @@ def make_voila_bundle( :param image: the optional docker image to be specified for off-host execution. Default = None. :return: a file-like object containing the bundle tarball. """ - extra_files = list(extra_files) if extra_files else [] - entrypoint_candidates = infer_entrypoint_candidates(path=abspath(path), mimetype="text/ipynb") - - if len(entrypoint_candidates) <= 0: - if entrypoint is None: - raise RSConnectException("No valid entrypoint found.") - elif len(entrypoint_candidates) == 1: - if entrypoint: - entrypoint = abs_entrypoint(path, entrypoint) - else: - entrypoint = entrypoint_candidates[0] - else: # len(entrypoint_candidates) > 1: - if entrypoint is None and not multi_notebook: - raise RSConnectException( - """ - Unable to infer entrypoint from multiple candidates. - Multi-notebook deployments need to be specified with the following: - 1) A directory as the path - 2) Set multi_notebook=True, - i.e. --multi-notebook - """ - ) - - deploy_dir = guess_deploy_dir(path, entrypoint) - if multi_notebook: - deploy_dir = entrypoint = abspath(path) - voila_json_path = join(deploy_dir, "voila.json") - if isfile(voila_json_path): - extra_files.append(voila_json_path) manifest = create_voila_manifest(**locals()) if manifest.data.get("files") is None: @@ -1055,7 +1035,7 @@ def make_voila_bundle( if multi_notebook and "metadata" in manifest_flattened_copy_data: manifest_flattened_copy_data["metadata"]["entrypoint"] = "" bundle.add_to_buffer("manifest.json", json.dumps(manifest_flattened_copy_data, indent=2)) - bundle.deploy_dir = deploy_dir + bundle.deploy_dir = manifest.deploy_dir return bundle.to_file() @@ -1587,22 +1567,45 @@ def create_voila_manifest( :return: the manifest data structure. """ extra_files = list(extra_files) if extra_files else [] - excludes = list(excludes) if excludes else [] - excludes.extend([environment.filename, "manifest.json"]) - deploy_dir = kwargs.get("deploy_dir") + entrypoint_candidates = infer_entrypoint_candidates(path=abspath(path), mimetype="text/ipynb") - if not deploy_dir: - deploy_dir = guess_deploy_dir(path, entrypoint) + if len(entrypoint_candidates) <= 0: + if entrypoint is None: + raise RSConnectException("No valid entrypoint provided or found.") + elif len(entrypoint_candidates) == 1: + if entrypoint: + entrypoint = abs_entrypoint(path, entrypoint) + else: + entrypoint = entrypoint_candidates[0] + else: # len(entrypoint_candidates) > 1: + if entrypoint is None and not multi_notebook: + raise RSConnectException( + """ + Unable to infer entrypoint from multiple candidates. + Multi-notebook deployments need to be specified with the following: + 1) A directory as the path + 2) Set multi_notebook=True, + i.e. include --multi-notebook (or -m) in the CLI command. + """ + ) + deploy_dir = guess_deploy_dir(path, entrypoint) + if multi_notebook: + deploy_dir = entrypoint = abspath(path) extra_files = validate_extra_files(deploy_dir, extra_files) + excludes = list(excludes) if excludes else [] + excludes.extend([environment.filename, "manifest.json"]) excludes.extend(list_environment_dirs(deploy_dir)) - manifest = Manifest(app_mode=AppModes.JUPYTER_VOILA, environment=environment, entrypoint=entrypoint, image=image) - if isfile(path): + voila_json_path = join(deploy_dir, "voila.json") + if isfile(voila_json_path): + extra_files.append(voila_json_path) + + manifest = Manifest(app_mode=AppModes.JUPYTER_VOILA, environment=environment, entrypoint=entrypoint, image=image) + manifest.deploy_dir = deploy_dir + if entrypoint and isfile(entrypoint): validate_file_is_notebook(entrypoint) manifest.entrypoint = entrypoint - else: - manifest.entrypoint = entrypoint or "" manifest.add_to_buffer(join(deploy_dir, environment.filename), environment.contents) From 5eccf3379b876252dbd730fac37cc4f9dfaf4397 Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Fri, 17 Feb 2023 11:32:23 -0500 Subject: [PATCH 82/97] relax entrypoint check for multi-notebook --- rsconnect/bundle.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/rsconnect/bundle.py b/rsconnect/bundle.py index 3dd6d22e..7fece924 100644 --- a/rsconnect/bundle.py +++ b/rsconnect/bundle.py @@ -1539,6 +1539,15 @@ def write_notebook_manifest_json( return exists(join(directory, environment.filename)) +MULTI_NOTEBOOK_EXC_MSG = """ +Unable to infer entrypoint. +Multi-notebook deployments need to be specified with the following: +1) A directory as the path +2) Set multi_notebook=True, + i.e. include --multi-notebook (or -m) in the CLI command. +""" + + def create_voila_manifest( path: str, entrypoint: str, @@ -1569,25 +1578,17 @@ def create_voila_manifest( extra_files = list(extra_files) if extra_files else [] entrypoint_candidates = infer_entrypoint_candidates(path=abspath(path), mimetype="text/ipynb") - if len(entrypoint_candidates) <= 0: + if len(entrypoint_candidates) <= 0 and not multi_notebook: if entrypoint is None: - raise RSConnectException("No valid entrypoint provided or found.") - elif len(entrypoint_candidates) == 1: + raise RSConnectException(MULTI_NOTEBOOK_EXC_MSG) + elif len(entrypoint_candidates) == 1 and not multi_notebook: if entrypoint: entrypoint = abs_entrypoint(path, entrypoint) else: entrypoint = entrypoint_candidates[0] else: # len(entrypoint_candidates) > 1: if entrypoint is None and not multi_notebook: - raise RSConnectException( - """ - Unable to infer entrypoint from multiple candidates. - Multi-notebook deployments need to be specified with the following: - 1) A directory as the path - 2) Set multi_notebook=True, - i.e. include --multi-notebook (or -m) in the CLI command. - """ - ) + raise RSConnectException(MULTI_NOTEBOOK_EXC_MSG) deploy_dir = guess_deploy_dir(path, entrypoint) if multi_notebook: From 8311a1ea6bf885d946578c4d453c76319835c668 Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Fri, 17 Feb 2023 13:34:32 -0500 Subject: [PATCH 83/97] no_args_is_help & update Connect version --- rsconnect/main.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rsconnect/main.py b/rsconnect/main.py index c90dbfe1..7b92c4da 100644 --- a/rsconnect/main.py +++ b/rsconnect/main.py @@ -847,8 +847,9 @@ def deploy_notebook( # noinspection SpellCheckingInspection,DuplicatedCode @deploy.command( name="voila", - short_help="Deploy Jupyter notebook in Voila mode to RStudio Connect [v2022.10.0+].", + short_help="Deploy Jupyter notebook in Voila mode to RStudio Connect [v2023.02.0+].", help=("Deploy a Jupyter notebook in Voila mode to RStudio Connect."), + no_args_is_help=True, ) @server_args @content_args From 0165f23100841d8a95f152db08cc1ac37c88ad3e Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Fri, 17 Feb 2023 13:54:41 -0500 Subject: [PATCH 84/97] More deploy_dir checks --- rsconnect/bundle.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/rsconnect/bundle.py b/rsconnect/bundle.py index 7fece924..abad213d 100644 --- a/rsconnect/bundle.py +++ b/rsconnect/bundle.py @@ -962,23 +962,29 @@ def make_html_bundle( def guess_deploy_dir(path, entrypoint): + if not path and not entrypoint: + raise RSConnectException("No path or entrypoint provided.") deploy_dir = None - if isfile(path): + if path and isfile(path): if not entrypoint: deploy_dir = dirname(abspath(path)) - if isfile(entrypoint) and path != entrypoint: + elif isfile(entrypoint) and path != entrypoint: raise RSConnectException("Path and entrypoint need to match if they are are files.") - if isdir(entrypoint): + elif isdir(entrypoint): raise RSConnectException("Entrypoint cannot be a directory while the path is a file.") - elif isdir(path): + elif path and isdir(path): if not entrypoint: deploy_dir = abspath(path) - elif isdir(entrypoint): + elif entrypoint and isdir(entrypoint): raise RSConnectException("Path and entrypoint cannot both be directories.") elif entrypoint: guess_entry_file = os.path.join(abspath(path), basename(entrypoint)) if isfile(guess_entry_file): deploy_dir = dirname(guess_entry_file) + elif isfile(entrypoint): + deploy_dir = dirname(abspath(entrypoint)) + elif not path and entrypoint: + raise RSConnectException("A path needs to be provided.") else: deploy_dir = abspath(path) return deploy_dir @@ -1578,6 +1584,7 @@ def create_voila_manifest( extra_files = list(extra_files) if extra_files else [] entrypoint_candidates = infer_entrypoint_candidates(path=abspath(path), mimetype="text/ipynb") + deploy_dir = guess_deploy_dir(path, entrypoint) if len(entrypoint_candidates) <= 0 and not multi_notebook: if entrypoint is None: raise RSConnectException(MULTI_NOTEBOOK_EXC_MSG) @@ -1590,7 +1597,6 @@ def create_voila_manifest( if entrypoint is None and not multi_notebook: raise RSConnectException(MULTI_NOTEBOOK_EXC_MSG) - deploy_dir = guess_deploy_dir(path, entrypoint) if multi_notebook: deploy_dir = entrypoint = abspath(path) extra_files = validate_extra_files(deploy_dir, extra_files) From 008bd2f2b9d8f01dc31247e5eeb8702ca9356853 Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Fri, 17 Feb 2023 16:47:15 -0500 Subject: [PATCH 85/97] update guess_deploy_dir --- rsconnect/bundle.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/rsconnect/bundle.py b/rsconnect/bundle.py index abad213d..586e81da 100644 --- a/rsconnect/bundle.py +++ b/rsconnect/bundle.py @@ -962,31 +962,35 @@ def make_html_bundle( def guess_deploy_dir(path, entrypoint): + abs_path = abspath(path) if path else None + abs_entrypoint = abspath(entrypoint) if entrypoint else None if not path and not entrypoint: raise RSConnectException("No path or entrypoint provided.") deploy_dir = None if path and isfile(path): if not entrypoint: - deploy_dir = dirname(abspath(path)) - elif isfile(entrypoint) and path != entrypoint: - raise RSConnectException("Path and entrypoint need to match if they are are files.") + deploy_dir = dirname(abs_path) + elif isfile(entrypoint) and abs_path != abs_entrypoint: + raise RSConnectException("Path and entrypoint need to match if they are both files.") + elif isfile(entrypoint) and abs_path == abs_entrypoint: + deploy_dir = dirname(abs_path) elif isdir(entrypoint): raise RSConnectException("Entrypoint cannot be a directory while the path is a file.") elif path and isdir(path): if not entrypoint: - deploy_dir = abspath(path) + deploy_dir = abs_path elif entrypoint and isdir(entrypoint): raise RSConnectException("Path and entrypoint cannot both be directories.") elif entrypoint: - guess_entry_file = os.path.join(abspath(path), basename(entrypoint)) + guess_entry_file = os.path.join(abs_path, basename(entrypoint)) if isfile(guess_entry_file): deploy_dir = dirname(guess_entry_file) elif isfile(entrypoint): - deploy_dir = dirname(abspath(entrypoint)) + deploy_dir = dirname(abs_entrypoint) elif not path and entrypoint: raise RSConnectException("A path needs to be provided.") else: - deploy_dir = abspath(path) + deploy_dir = abs_path return deploy_dir From 9ca201109ecffb44423b092f271a8bde37c55bce Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Mon, 20 Feb 2023 13:37:56 -0500 Subject: [PATCH 86/97] add voila test files --- tests/testdata/voila/bqplot/bqplot.ipynb | 62 ++++++++++++++++++++ tests/testdata/voila/bqplot/requirements.txt | 1 + 2 files changed, 63 insertions(+) create mode 100644 tests/testdata/voila/bqplot/bqplot.ipynb create mode 100644 tests/testdata/voila/bqplot/requirements.txt diff --git a/tests/testdata/voila/bqplot/bqplot.ipynb b/tests/testdata/voila/bqplot/bqplot.ipynb new file mode 100644 index 00000000..66ebe130 --- /dev/null +++ b/tests/testdata/voila/bqplot/bqplot.ipynb @@ -0,0 +1,62 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# So easy, *voilà*!\n", + "\n", + "In this example notebook, we demonstrate how Voilà can render custom Jupyter widgets such as [bqplot](https://github.com/bloomberg/bqplot). " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import warnings\n", + "warnings.filterwarnings('ignore')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "from bqplot import pyplot as plt\n", + "\n", + "plt.figure(1, title='Line Chart')\n", + "np.random.seed(0)\n", + "n = 200\n", + "x = np.linspace(0.0, 10.0, n)\n", + "y = np.cumsum(np.random.randn(n))\n", + "plt.plot(x, y)\n", + "plt.show()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.12" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/tests/testdata/voila/bqplot/requirements.txt b/tests/testdata/voila/bqplot/requirements.txt new file mode 100644 index 00000000..35c00ee4 --- /dev/null +++ b/tests/testdata/voila/bqplot/requirements.txt @@ -0,0 +1 @@ +bqplot From 23ec32da28d7fef9b78a4c7419bb3cc67f4e4e45 Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Mon, 20 Feb 2023 13:46:46 -0500 Subject: [PATCH 87/97] remove exception in infer_entrypoint_candidates --- rsconnect/bundle.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/rsconnect/bundle.py b/rsconnect/bundle.py index 586e81da..2565d9e4 100644 --- a/rsconnect/bundle.py +++ b/rsconnect/bundle.py @@ -909,10 +909,12 @@ def infer_entrypoint(path, mimetype): def infer_entrypoint_candidates(path, mimetype) -> List: + if not path: + return [] if isfile(path): return [path] if not isdir(path): - raise RSConnectException("Entrypoint is not a valid file type or directory.") + return [] default_mimetype_entrypoints = defaultdict(str) default_mimetype_entrypoints["text/html"] = "index.html" From e2a047649fd897314cdc08121453782a7a9074ff Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Mon, 20 Feb 2023 13:50:58 -0500 Subject: [PATCH 88/97] add exception if Manifest cannot be flattened --- rsconnect/bundle.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/rsconnect/bundle.py b/rsconnect/bundle.py index 2565d9e4..ce5d827b 100644 --- a/rsconnect/bundle.py +++ b/rsconnect/bundle.py @@ -181,6 +181,8 @@ def discard_from_buffer(self, key): @property def flattened_data(self): + if self.entrypoint is None: + raise RSConnectException("A valid entrypoint must be provided.") new_data_files = {} deploy_dir = dirname(self.entrypoint) if isfile(self.entrypoint) else self.entrypoint for path in self.data["files"]: @@ -190,6 +192,8 @@ def flattened_data(self): @property def flattened_buffer(self): + if self.entrypoint is None: + raise RSConnectException("A valid entrypoint must be provided.") new_buffer = {} deploy_dir = dirname(self.entrypoint) if isfile(self.entrypoint) else self.entrypoint for k, v in self.buffer.items(): @@ -199,10 +203,14 @@ def flattened_buffer(self): @property def flattened_entrypoint(self): + if self.entrypoint is None: + raise RSConnectException("A valid entrypoint must be provided.") return relpath(self.entrypoint, dirname(self.entrypoint)) @property def flattened_copy(self): + if self.entrypoint is None: + raise RSConnectException("A valid entrypoint must be provided.") new_manifest = deepcopy(self) new_manifest.data["files"] = self.flattened_data new_manifest.buffer = self.flattened_buffer From 96b61d707145a5079f8b6bc09f39d55c52f87025 Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Mon, 20 Feb 2023 13:53:56 -0500 Subject: [PATCH 89/97] check valid path in create_voila_manifest --- rsconnect/bundle.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/rsconnect/bundle.py b/rsconnect/bundle.py index ce5d827b..d9ed5cde 100644 --- a/rsconnect/bundle.py +++ b/rsconnect/bundle.py @@ -1595,6 +1595,8 @@ def create_voila_manifest( :param image: the optional docker image to be specified for off-host execution. Default = None. :return: the manifest data structure. """ + if not path: + raise RSConnectException("A valid path must be provided.") extra_files = list(extra_files) if extra_files else [] entrypoint_candidates = infer_entrypoint_candidates(path=abspath(path), mimetype="text/ipynb") From d111e8eb12800ad81d3af699fb55b6a540e7d09e Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Mon, 20 Feb 2023 15:52:57 -0500 Subject: [PATCH 90/97] update multi_notebook check --- rsconnect/bundle.py | 34 +++++++++++++++++++++++----------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/rsconnect/bundle.py b/rsconnect/bundle.py index d9ed5cde..c12ac0a3 100644 --- a/rsconnect/bundle.py +++ b/rsconnect/bundle.py @@ -1305,6 +1305,14 @@ def validate_entry_point(entry_point, directory): return entry_point +def _warn_on_ignored_entrypoint(entrypoint): + if entrypoint: + click.secho( + " Warning: entrypoint will not be used or considered for multi-notebook mode.", + fg="yellow", + ) + + def _warn_on_ignored_manifest(directory): """ Checks for the existence of a file called manifest.json in the given directory. @@ -1601,19 +1609,23 @@ def create_voila_manifest( entrypoint_candidates = infer_entrypoint_candidates(path=abspath(path), mimetype="text/ipynb") deploy_dir = guess_deploy_dir(path, entrypoint) - if len(entrypoint_candidates) <= 0 and not multi_notebook: - if entrypoint is None: - raise RSConnectException(MULTI_NOTEBOOK_EXC_MSG) - elif len(entrypoint_candidates) == 1 and not multi_notebook: - if entrypoint: - entrypoint = abs_entrypoint(path, entrypoint) - else: - entrypoint = entrypoint_candidates[0] - else: # len(entrypoint_candidates) > 1: - if entrypoint is None and not multi_notebook: - raise RSConnectException(MULTI_NOTEBOOK_EXC_MSG) + if not multi_notebook: + if len(entrypoint_candidates) <= 0: + if entrypoint is None: + raise RSConnectException(MULTI_NOTEBOOK_EXC_MSG) + elif len(entrypoint_candidates) == 1: + if entrypoint: + entrypoint = abs_entrypoint(path, entrypoint) + else: + entrypoint = entrypoint_candidates[0] + else: # len(entrypoint_candidates) > 1: + if entrypoint is None: + raise RSConnectException(MULTI_NOTEBOOK_EXC_MSG) if multi_notebook: + if path and not isdir(path): + raise RSConnectException(MULTI_NOTEBOOK_EXC_MSG) + _warn_on_ignored_entrypoint(entrypoint) deploy_dir = entrypoint = abspath(path) extra_files = validate_extra_files(deploy_dir, extra_files) excludes = list(excludes) if excludes else [] From a092e5890c609d4e1ed26ef2376ca36ae9fff50e Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Mon, 20 Feb 2023 16:37:52 -0500 Subject: [PATCH 91/97] add voila multi-notebook test files --- .../voila/multi-voila/bqplot/bqplot.ipynb | 62 ++++++++ .../multi-voila/dashboard/dashboard.ipynb | 148 ++++++++++++++++++ .../voila/multi-voila/requirements.txt | 4 + 3 files changed, 214 insertions(+) create mode 100644 tests/testdata/voila/multi-voila/bqplot/bqplot.ipynb create mode 100644 tests/testdata/voila/multi-voila/dashboard/dashboard.ipynb create mode 100644 tests/testdata/voila/multi-voila/requirements.txt diff --git a/tests/testdata/voila/multi-voila/bqplot/bqplot.ipynb b/tests/testdata/voila/multi-voila/bqplot/bqplot.ipynb new file mode 100644 index 00000000..92b7d19d --- /dev/null +++ b/tests/testdata/voila/multi-voila/bqplot/bqplot.ipynb @@ -0,0 +1,62 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# So easy, *voilà*!\n", + "\n", + "In this example notebook, we demonstrate how Voilà can render custom Jupyter widgets such as [bqplot](https://github.com/bloomberg/bqplot). " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import warnings\n", + "warnings.filterwarnings('ignore')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "from bqplot import pyplot as plt\n", + "\n", + "plt.figure(1, title='Line Chart')\n", + "np.random.seed(0)\n", + "n = 200\n", + "x = np.linspace(0.0, 10.0, n)\n", + "y = np.cumsum(np.random.randn(n))\n", + "plt.plot(x, y)\n", + "plt.show()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/tests/testdata/voila/multi-voila/dashboard/dashboard.ipynb b/tests/testdata/voila/multi-voila/dashboard/dashboard.ipynb new file mode 100644 index 00000000..cf0c2bce --- /dev/null +++ b/tests/testdata/voila/multi-voila/dashboard/dashboard.ipynb @@ -0,0 +1,148 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This demo uses Voilà to render a notebook to a custom HTML page using gridstack.js for the layout of each output. In the cell metadata you can change the default cell with and height (in grid units between 1 and 12) by specifying.\n", + " * `grid_row`\n", + " * `grid_columns`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "n = 200\n", + "\n", + "x = np.linspace(0.0, 10.0, n)\n", + "y = np.cumsum(np.random.randn(n)*10).astype(int)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import ipywidgets as widgets" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "label_selected = widgets.Label(value=\"Selected: 0\")\n", + "label_selected" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "grid_columns": 8, + "grid_rows": 4 + }, + "outputs": [], + "source": [ + "import numpy as np\n", + "from bqplot import pyplot as plt\n", + "import bqplot\n", + "\n", + "fig = plt.figure( title='Histogram')\n", + "np.random.seed(0)\n", + "hist = plt.hist(y, bins=25)\n", + "hist.scales['sample'].min = float(y.min())\n", + "hist.scales['sample'].max = float(y.max())\n", + "display(fig)\n", + "fig.layout.width = 'auto'\n", + "fig.layout.height = 'auto'\n", + "fig.layout.min_height = '300px' # so it shows nicely in the notebook\n", + "fig.layout.flex = '1'" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "grid_columns": 12, + "grid_rows": 6 + }, + "outputs": [], + "source": [ + "import numpy as np\n", + "from bqplot import pyplot as plt\n", + "import bqplot\n", + "\n", + "fig = plt.figure( title='Line Chart')\n", + "np.random.seed(0)\n", + "n = 200\n", + "p = plt.plot(x, y)\n", + "fig" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "fig.layout.width = 'auto'\n", + "fig.layout.height = 'auto'\n", + "fig.layout.min_height = '300px' # so it shows nicely in the notebook\n", + "fig.layout.flex = '1'" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "brushintsel = bqplot.interacts.BrushIntervalSelector(scale=p.scales['x'])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def update_range(*args):\n", + " label_selected.value = \"Selected range {}\".format(brushintsel.selected)\n", + " mask = (x > brushintsel.selected[0]) & (x < brushintsel.selected[1])\n", + " hist.sample = y[mask]\n", + " \n", + "brushintsel.observe(update_range, 'selected')\n", + "fig.interaction = brushintsel" + ] + } + ], + "metadata": { + "celltoolbar": "Edit Metadata", + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.4" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/tests/testdata/voila/multi-voila/requirements.txt b/tests/testdata/voila/multi-voila/requirements.txt new file mode 100644 index 00000000..9d6d6ab4 --- /dev/null +++ b/tests/testdata/voila/multi-voila/requirements.txt @@ -0,0 +1,4 @@ +numpy +ipywidgets +bqplot + From a74cf5b570b669b94d13922d7a0410b1c4fba1fd Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Mon, 20 Feb 2023 18:36:49 -0500 Subject: [PATCH 92/97] add voila unit tests --- tests/test_bundle.py | 425 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 424 insertions(+), 1 deletion(-) diff --git a/tests/test_bundle.py b/tests/test_bundle.py index fce50af3..962c8817 100644 --- a/tests/test_bundle.py +++ b/tests/test_bundle.py @@ -8,7 +8,7 @@ import tempfile import pytest from unittest import TestCase -from os.path import dirname, join, basename +from os.path import dirname, join, basename, abspath from rsconnect.bundle import ( _default_title, @@ -21,6 +21,7 @@ make_notebook_html_bundle, make_notebook_source_bundle, keep_manifest_specified_file, + make_voila_bundle, to_bytes, make_source_manifest, make_quarto_manifest, @@ -28,6 +29,9 @@ validate_entry_point, validate_extra_files, which_python, + guess_deploy_dir, + Manifest, + create_voila_manifest, ) import rsconnect.bundle from rsconnect.exception import RSConnectException @@ -801,3 +805,422 @@ def test_is_not_executable(self): with tempfile.NamedTemporaryFile() as tmpfile: with self.assertRaises(RSConnectException): which_python(tmpfile.name) + + +cur_dir = os.path.dirname(__file__) +bqplot_dir = os.path.join(cur_dir, "./testdata/voila/bqplot/") +bqplot_ipynb = os.path.join(bqplot_dir, "bqplot.ipynb") +multivoila_dir = os.path.join(cur_dir, "./testdata/voila/multi-voila/") + + +class Test_guess_deploy_dir(TestCase): + def test_guess_deploy_dir(self): + with self.assertRaises(RSConnectException): + guess_deploy_dir(None, None) + with self.assertRaises(RSConnectException): + guess_deploy_dir(None, bqplot_dir) + with self.assertRaises(RSConnectException): + guess_deploy_dir(bqplot_dir, bqplot_dir) + self.assertEqual(abspath(bqplot_dir), guess_deploy_dir(bqplot_dir, None)) + self.assertEqual(abspath(bqplot_dir), guess_deploy_dir(bqplot_ipynb, None)) + self.assertEqual(abspath(bqplot_dir), guess_deploy_dir(bqplot_ipynb, bqplot_ipynb)) + self.assertEqual(abspath(bqplot_dir), guess_deploy_dir(bqplot_dir, "bqplot.ipynb")) + + +@pytest.mark.parametrize( + ( + "path", + "entrypoint", + ), + [ + ( + None, + None, + ), + ( + None, + bqplot_ipynb, + ), + ( + bqplot_dir, + bqplot_dir, + ), + ( + bqplot_dir, + None, + ), + ( + bqplot_ipynb, + None, + ), + ( + bqplot_ipynb, + bqplot_ipynb, + ), + ( + bqplot_dir, + bqplot_ipynb, + ), + ], +) +def test_create_voila_manifest(path, entrypoint): + environment = Environment( + conda=None, + contents="bqplot\n", + error=None, + filename="requirements.txt", + locale="en_US.UTF-8", + package_manager="pip", + pip="23.0", + python="3.8.12", + source="file", + ) + ans = { + "version": 1, + "locale": "en_US.UTF-8", + "metadata": {"appmode": "jupyter-voila", "entrypoint": "bqplot.ipynb"}, + "python": { + "version": "3.8.12", + "package_manager": {"name": "pip", "version": "23.0", "package_file": "requirements.txt"}, + }, + "files": { + "requirements.txt": {"checksum": "9cce1aac313043abd5690f67f84338ed"}, + "bqplot.ipynb": {"checksum": "79f8622228eded646a3038848de5ffd9"}, + }, + } + manifest = Manifest() + if (path, entrypoint) in ( + (None, None), + (None, bqplot_ipynb), + (bqplot_dir, bqplot_dir), + ): + with pytest.raises(RSConnectException) as _: + manifest = create_voila_manifest( + path, + entrypoint, + environment, + app_mode=AppModes.JUPYTER_VOILA, + extra_files=None, + excludes=None, + force_generate=True, + image=None, + multi_notebook=False, + ) + else: + manifest = create_voila_manifest( + path, + entrypoint, + environment, + app_mode=AppModes.JUPYTER_VOILA, + extra_files=None, + excludes=None, + force_generate=True, + image=None, + multi_notebook=False, + ) + assert ans == json.loads(manifest.flattened_copy.json) + + +@pytest.mark.parametrize( + ( + "path", + "entrypoint", + ), + [ + ( + None, + None, + ), + ( + None, + bqplot_ipynb, + ), + ( + multivoila_dir, + multivoila_dir, + ), + ( + multivoila_dir, + None, + ), + ( + bqplot_ipynb, + None, + ), + ( + bqplot_ipynb, + bqplot_ipynb, + ), + ( + multivoila_dir, + bqplot_ipynb, + ), + ], +) +def test_create_voila_manifest_multi_notebook(path, entrypoint): + environment = Environment( + conda=None, + contents="bqplot\n", + error=None, + filename="requirements.txt", + locale="en_US.UTF-8", + package_manager="pip", + pip="23.0", + python="3.8.12", + source="file", + ) + ans = { + "version": 1, + "locale": "en_US.UTF-8", + "metadata": {"appmode": "jupyter-voila", "entrypoint": "multi-voila"}, + "python": { + "version": "3.8.12", + "package_manager": {"name": "pip", "version": "23.0", "package_file": "requirements.txt"}, + }, + "files": { + "requirements.txt": {"checksum": "9cce1aac313043abd5690f67f84338ed"}, + "bqplot/bqplot.ipynb": {"checksum": "9f283b29889500e6c78e83ad1257e03f"}, + "dashboard/dashboard.ipynb": {"checksum": "6b42a0730d61e5344a3e734f5bbeec25"}, + }, + } + manifest = Manifest() + if (path, entrypoint) in ( + (None, None), + (None, bqplot_ipynb), + (multivoila_dir, multivoila_dir), + (bqplot_ipynb, None), + (bqplot_ipynb, bqplot_ipynb), + ): + with pytest.raises(RSConnectException) as _: + manifest = create_voila_manifest( + path, + entrypoint, + environment, + app_mode=AppModes.JUPYTER_VOILA, + extra_files=None, + excludes=None, + force_generate=True, + image=None, + multi_notebook=True, + ) + else: + manifest = create_voila_manifest( + path, + entrypoint, + environment, + app_mode=AppModes.JUPYTER_VOILA, + extra_files=None, + excludes=None, + force_generate=True, + image=None, + multi_notebook=True, + ) + assert ans == json.loads(manifest.flattened_copy.json) + + +@pytest.mark.parametrize( + ( + "path", + "entrypoint", + ), + [ + ( + None, + None, + ), + ( + None, + bqplot_ipynb, + ), + ( + bqplot_dir, + bqplot_dir, + ), + ( + bqplot_dir, + None, + ), + ( + bqplot_ipynb, + None, + ), + ( + bqplot_ipynb, + bqplot_ipynb, + ), + ( + bqplot_dir, + bqplot_ipynb, + ), + ], +) +def test_make_voila_bundle( + path, + entrypoint, +): + environment = Environment( + conda=None, + contents="bqplot", + error=None, + filename="requirements.txt", + locale="en_US.UTF-8", + package_manager="pip", + pip="23.0", + python="3.8.12", + source="file", + ) + ans = { + "version": 1, + "locale": "en_US.UTF-8", + "metadata": {"appmode": "jupyter-voila", "entrypoint": "bqplot.ipynb"}, + "python": { + "version": "3.8.12", + "package_manager": {"name": "pip", "version": "23.0", "package_file": "requirements.txt"}, + }, + "files": { + "requirements.txt": {"checksum": "9395f3162b7779c57c86b187fa441d96"}, + "bqplot.ipynb": {"checksum": "79f8622228eded646a3038848de5ffd9"}, + }, + } + if (path, entrypoint) in ( + (None, None), + (None, bqplot_ipynb), + (bqplot_dir, bqplot_dir), + ): + with pytest.raises(RSConnectException) as _: + bundle = make_voila_bundle( + path, + entrypoint, + extra_files=None, + excludes=None, + force_generate=True, + environment=environment, + image=None, + multi_notebook=False, + ) + else: + with make_voila_bundle( + path, + entrypoint, + extra_files=None, + excludes=None, + force_generate=True, + environment=environment, + image=None, + multi_notebook=False, + ) as bundle, tarfile.open(mode="r:gz", fileobj=bundle) as tar: + names = sorted(tar.getnames()) + assert names == [ + "bqplot.ipynb", + "manifest.json", + "requirements.txt", + ] + reqs = tar.extractfile("requirements.txt").read() + assert reqs == b"bqplot" + assert ans == json.loads(tar.extractfile("manifest.json").read().decode("utf-8")) + + +@pytest.mark.parametrize( + ( + "path", + "entrypoint", + ), + [ + ( + None, + None, + ), + ( + None, + bqplot_ipynb, + ), + ( + multivoila_dir, + multivoila_dir, + ), + ( + multivoila_dir, + None, + ), + ( + bqplot_ipynb, + None, + ), + ( + bqplot_ipynb, + bqplot_ipynb, + ), + ( + multivoila_dir, + bqplot_ipynb, + ), + ], +) +def test_make_voila_bundle_multi_notebook( + path, + entrypoint, +): + environment = Environment( + conda=None, + contents="bqplot", + error=None, + filename="requirements.txt", + locale="en_US.UTF-8", + package_manager="pip", + pip="23.0", + python="3.8.12", + source="file", + ) + ans = { + "version": 1, + "locale": "en_US.UTF-8", + "metadata": {"appmode": "jupyter-voila", "entrypoint": ""}, + "python": { + "version": "3.8.12", + "package_manager": {"name": "pip", "version": "23.0", "package_file": "requirements.txt"}, + }, + "files": { + "requirements.txt": {"checksum": "9395f3162b7779c57c86b187fa441d96"}, + "bqplot/bqplot.ipynb": {"checksum": "9f283b29889500e6c78e83ad1257e03f"}, + "dashboard/dashboard.ipynb": {"checksum": "6b42a0730d61e5344a3e734f5bbeec25"}, + }, + } + if (path, entrypoint) in ( + (None, None), + (None, bqplot_ipynb), + (multivoila_dir, multivoila_dir), + (bqplot_ipynb, None), + (bqplot_ipynb, bqplot_ipynb), + ): + with pytest.raises(RSConnectException) as _: + bundle = make_voila_bundle( + path, + entrypoint, + extra_files=None, + excludes=None, + force_generate=True, + environment=environment, + image=None, + multi_notebook=True, + ) + else: + with make_voila_bundle( + path, + entrypoint, + extra_files=None, + excludes=None, + force_generate=True, + environment=environment, + image=None, + multi_notebook=True, + ) as bundle, tarfile.open(mode="r:gz", fileobj=bundle) as tar: + names = sorted(tar.getnames()) + assert names == [ + "bqplot/bqplot.ipynb", + "dashboard/dashboard.ipynb", + "manifest.json", + "requirements.txt", + ] + reqs = tar.extractfile("requirements.txt").read() + assert reqs == b"bqplot" + assert ans == json.loads(tar.extractfile("manifest.json").read().decode("utf-8")) From 294991c11ac32cebd0a2d8d5961cdc14871f3a54 Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Fri, 24 Feb 2023 11:06:36 -0500 Subject: [PATCH 93/97] add more voila test folder --- tests/testdata/voila/dashboard/bqplot.ipynb | 62 ++++++++ .../testdata/voila/dashboard/dashboard.ipynb | 148 ++++++++++++++++++ .../testdata/voila/dashboard/requirements.txt | 4 + 3 files changed, 214 insertions(+) create mode 100644 tests/testdata/voila/dashboard/bqplot.ipynb create mode 100644 tests/testdata/voila/dashboard/dashboard.ipynb create mode 100644 tests/testdata/voila/dashboard/requirements.txt diff --git a/tests/testdata/voila/dashboard/bqplot.ipynb b/tests/testdata/voila/dashboard/bqplot.ipynb new file mode 100644 index 00000000..66ebe130 --- /dev/null +++ b/tests/testdata/voila/dashboard/bqplot.ipynb @@ -0,0 +1,62 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# So easy, *voilà*!\n", + "\n", + "In this example notebook, we demonstrate how Voilà can render custom Jupyter widgets such as [bqplot](https://github.com/bloomberg/bqplot). " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import warnings\n", + "warnings.filterwarnings('ignore')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "from bqplot import pyplot as plt\n", + "\n", + "plt.figure(1, title='Line Chart')\n", + "np.random.seed(0)\n", + "n = 200\n", + "x = np.linspace(0.0, 10.0, n)\n", + "y = np.cumsum(np.random.randn(n))\n", + "plt.plot(x, y)\n", + "plt.show()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.12" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/tests/testdata/voila/dashboard/dashboard.ipynb b/tests/testdata/voila/dashboard/dashboard.ipynb new file mode 100644 index 00000000..cf0c2bce --- /dev/null +++ b/tests/testdata/voila/dashboard/dashboard.ipynb @@ -0,0 +1,148 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This demo uses Voilà to render a notebook to a custom HTML page using gridstack.js for the layout of each output. In the cell metadata you can change the default cell with and height (in grid units between 1 and 12) by specifying.\n", + " * `grid_row`\n", + " * `grid_columns`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "n = 200\n", + "\n", + "x = np.linspace(0.0, 10.0, n)\n", + "y = np.cumsum(np.random.randn(n)*10).astype(int)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import ipywidgets as widgets" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "label_selected = widgets.Label(value=\"Selected: 0\")\n", + "label_selected" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "grid_columns": 8, + "grid_rows": 4 + }, + "outputs": [], + "source": [ + "import numpy as np\n", + "from bqplot import pyplot as plt\n", + "import bqplot\n", + "\n", + "fig = plt.figure( title='Histogram')\n", + "np.random.seed(0)\n", + "hist = plt.hist(y, bins=25)\n", + "hist.scales['sample'].min = float(y.min())\n", + "hist.scales['sample'].max = float(y.max())\n", + "display(fig)\n", + "fig.layout.width = 'auto'\n", + "fig.layout.height = 'auto'\n", + "fig.layout.min_height = '300px' # so it shows nicely in the notebook\n", + "fig.layout.flex = '1'" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "grid_columns": 12, + "grid_rows": 6 + }, + "outputs": [], + "source": [ + "import numpy as np\n", + "from bqplot import pyplot as plt\n", + "import bqplot\n", + "\n", + "fig = plt.figure( title='Line Chart')\n", + "np.random.seed(0)\n", + "n = 200\n", + "p = plt.plot(x, y)\n", + "fig" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "fig.layout.width = 'auto'\n", + "fig.layout.height = 'auto'\n", + "fig.layout.min_height = '300px' # so it shows nicely in the notebook\n", + "fig.layout.flex = '1'" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "brushintsel = bqplot.interacts.BrushIntervalSelector(scale=p.scales['x'])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def update_range(*args):\n", + " label_selected.value = \"Selected range {}\".format(brushintsel.selected)\n", + " mask = (x > brushintsel.selected[0]) & (x < brushintsel.selected[1])\n", + " hist.sample = y[mask]\n", + " \n", + "brushintsel.observe(update_range, 'selected')\n", + "fig.interaction = brushintsel" + ] + } + ], + "metadata": { + "celltoolbar": "Edit Metadata", + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.4" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/tests/testdata/voila/dashboard/requirements.txt b/tests/testdata/voila/dashboard/requirements.txt new file mode 100644 index 00000000..9d6d6ab4 --- /dev/null +++ b/tests/testdata/voila/dashboard/requirements.txt @@ -0,0 +1,4 @@ +numpy +ipywidgets +bqplot + From 82315e5b99be5090d2d11c751b039055647add0c Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Fri, 24 Feb 2023 11:11:50 -0500 Subject: [PATCH 94/97] fix entrypoint not made with abspath happened when path and entrypoint were provided and there were multiple notebooks in the deploy_dir --- rsconnect/bundle.py | 1 + tests/test_bundle.py | 56 +++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 56 insertions(+), 1 deletion(-) diff --git a/rsconnect/bundle.py b/rsconnect/bundle.py index c12ac0a3..d0b73707 100644 --- a/rsconnect/bundle.py +++ b/rsconnect/bundle.py @@ -1621,6 +1621,7 @@ def create_voila_manifest( else: # len(entrypoint_candidates) > 1: if entrypoint is None: raise RSConnectException(MULTI_NOTEBOOK_EXC_MSG) + entrypoint = abs_entrypoint(path, entrypoint) if multi_notebook: if path and not isdir(path): diff --git a/tests/test_bundle.py b/tests/test_bundle.py index 962c8817..a56134ca 100644 --- a/tests/test_bundle.py +++ b/tests/test_bundle.py @@ -810,6 +810,8 @@ def test_is_not_executable(self): cur_dir = os.path.dirname(__file__) bqplot_dir = os.path.join(cur_dir, "./testdata/voila/bqplot/") bqplot_ipynb = os.path.join(bqplot_dir, "bqplot.ipynb") +dashboard_dir = os.path.join(cur_dir, "./testdata/voila/dashboard/") +dashboard_ipynb = os.path.join(dashboard_dir, "dashboard.ipynb") multivoila_dir = os.path.join(cur_dir, "./testdata/voila/multi-voila/") @@ -863,7 +865,7 @@ def test_guess_deploy_dir(self): ), ], ) -def test_create_voila_manifest(path, entrypoint): +def test_create_voila_manifest_1(path, entrypoint): environment = Environment( conda=None, contents="bqplot\n", @@ -921,6 +923,58 @@ def test_create_voila_manifest(path, entrypoint): assert ans == json.loads(manifest.flattened_copy.json) +@pytest.mark.parametrize( + ( + "path", + "entrypoint", + ), + [ + ( + dashboard_dir, + dashboard_ipynb, + ), + ], +) +def test_create_voila_manifest_2(path, entrypoint): + environment = Environment( + conda=None, + contents="numpy\nipywidgets\nbqplot\n", + error=None, + filename="requirements.txt", + locale="en_US.UTF-8", + package_manager="pip", + pip="23.0", + python="3.8.12", + source="file", + ) + ans = { + "version": 1, + "locale": "en_US.UTF-8", + "metadata": {"appmode": "jupyter-voila", "entrypoint": "dashboard.ipynb"}, + "python": { + "version": "3.8.12", + "package_manager": {"name": "pip", "version": "23.0", "package_file": "requirements.txt"}, + }, + "files": { + "requirements.txt": {"checksum": "d51994456975ff487749acc247ae6d63"}, + "bqplot.ipynb": {"checksum": "79f8622228eded646a3038848de5ffd9"}, + "dashboard.ipynb": {"checksum": "6b42a0730d61e5344a3e734f5bbeec25"}, + }, + } + manifest = create_voila_manifest( + path, + entrypoint, + environment, + app_mode=AppModes.JUPYTER_VOILA, + extra_files=None, + excludes=None, + force_generate=True, + image=None, + multi_notebook=False, + ) + assert ans == json.loads(manifest.flattened_copy.json) + + @pytest.mark.parametrize( ( "path", From 5dddfcbc2a4daf2cc6d324d078592d21516fcf15 Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Fri, 24 Feb 2023 11:21:03 -0500 Subject: [PATCH 95/97] add more voila bundle test --- tests/test_bundle.py | 63 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/tests/test_bundle.py b/tests/test_bundle.py index a56134ca..834b97e8 100644 --- a/tests/test_bundle.py +++ b/tests/test_bundle.py @@ -1278,3 +1278,66 @@ def test_make_voila_bundle_multi_notebook( reqs = tar.extractfile("requirements.txt").read() assert reqs == b"bqplot" assert ans == json.loads(tar.extractfile("manifest.json").read().decode("utf-8")) + + +@pytest.mark.parametrize( + ( + "path", + "entrypoint", + ), + [ + ( + dashboard_dir, + dashboard_ipynb, + ), + ], +) +def test_make_voila_bundle_2( + path, + entrypoint, +): + environment = Environment( + conda=None, + contents="numpy\nipywidgets\nbqplot\n", + error=None, + filename="requirements.txt", + locale="en_US.UTF-8", + package_manager="pip", + pip="23.0", + python="3.8.12", + source="file", + ) + ans = { + "version": 1, + "locale": "en_US.UTF-8", + "metadata": {"appmode": "jupyter-voila", "entrypoint": "dashboard.ipynb"}, + "python": { + "version": "3.8.12", + "package_manager": {"name": "pip", "version": "23.0", "package_file": "requirements.txt"}, + }, + "files": { + "requirements.txt": {"checksum": "d51994456975ff487749acc247ae6d63"}, + "bqplot.ipynb": {"checksum": "79f8622228eded646a3038848de5ffd9"}, + "dashboard.ipynb": {"checksum": "6b42a0730d61e5344a3e734f5bbeec25"}, + }, + } + with make_voila_bundle( + path, + entrypoint, + extra_files=None, + excludes=None, + force_generate=True, + environment=environment, + image=None, + multi_notebook=False, + ) as bundle, tarfile.open(mode="r:gz", fileobj=bundle) as tar: + names = sorted(tar.getnames()) + assert names == [ + "bqplot.ipynb", + "dashboard.ipynb", + "manifest.json", + "requirements.txt", + ] + reqs = tar.extractfile("requirements.txt").read() + assert reqs == b"numpy\nipywidgets\nbqplot\n" + assert ans == json.loads(tar.extractfile("manifest.json").read().decode("utf-8")) From 6a2973ad6d59017a5e54b2287521282df2e7ca81 Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Fri, 24 Feb 2023 11:42:06 -0500 Subject: [PATCH 96/97] add abs_entrypoint when entrypoint_candidates==0 Most of the time this should raise exception. Adding this line should at least allow the bundle to be created. --- rsconnect/bundle.py | 1 + 1 file changed, 1 insertion(+) diff --git a/rsconnect/bundle.py b/rsconnect/bundle.py index d0b73707..235f4fc0 100644 --- a/rsconnect/bundle.py +++ b/rsconnect/bundle.py @@ -1613,6 +1613,7 @@ def create_voila_manifest( if len(entrypoint_candidates) <= 0: if entrypoint is None: raise RSConnectException(MULTI_NOTEBOOK_EXC_MSG) + entrypoint = abs_entrypoint(path, entrypoint) elif len(entrypoint_candidates) == 1: if entrypoint: entrypoint = abs_entrypoint(path, entrypoint) From 60fe87f32563cd6141004b92466b1acea02098c0 Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Wed, 1 Mar 2023 14:02:44 -0500 Subject: [PATCH 97/97] Update Connect version required to v2023.03.0+ --- rsconnect/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rsconnect/main.py b/rsconnect/main.py index 7b92c4da..b4f1f7b6 100644 --- a/rsconnect/main.py +++ b/rsconnect/main.py @@ -847,7 +847,7 @@ def deploy_notebook( # noinspection SpellCheckingInspection,DuplicatedCode @deploy.command( name="voila", - short_help="Deploy Jupyter notebook in Voila mode to RStudio Connect [v2023.02.0+].", + short_help="Deploy Jupyter notebook in Voila mode to RStudio Connect [v2023.03.0+].", help=("Deploy a Jupyter notebook in Voila mode to RStudio Connect."), no_args_is_help=True, )