From 6d436f30546d2a712002312e8931d25b3acb021c Mon Sep 17 00:00:00 2001 From: tdstein Date: Wed, 8 Feb 2023 15:10:53 -0500 Subject: [PATCH 01/16] Adds CONNECT_REQUEST_TIMEOUT env variable. Adds support for configurable HTTP and HTTPS timeouts via the environment variable "CONNECT_REQUEST_TIMEOUT". When not set, a default value of 300 seconds is used. --- rsconnect/actions_content.py | 8 +++---- rsconnect/api.py | 24 ++++++++++----------- rsconnect/http_support.py | 16 ++++++++------ rsconnect/timeouts.py | 41 ++++++++++++++++++++++++++++++++++++ tests/test_http_support.py | 2 +- tests/test_timeouts.py | 33 +++++++++++++++++++++++++++++ 6 files changed, 100 insertions(+), 24 deletions(-) create mode 100644 rsconnect/timeouts.py create mode 100644 tests/test_timeouts.py diff --git a/rsconnect/actions_content.py b/rsconnect/actions_content.py index 906d3566..42f67307 100644 --- a/rsconnect/actions_content.py +++ b/rsconnect/actions_content.py @@ -35,7 +35,7 @@ def build_add_content(connect_server, content_guids_with_bundle): + "please wait for it to finish before adding new content." ) - with RSConnectClient(connect_server, timeout=120) as client: + with RSConnectClient(connect_server) as client: if len(content_guids_with_bundle) == 1: all_content = [client.content_get(content_guids_with_bundle[0].guid)] else: @@ -290,7 +290,7 @@ def download_bundle(connect_server, guid_with_bundle): """ :param guid_with_bundle: models.ContentGuidWithBundle """ - with RSConnectClient(connect_server, timeout=120) as client: + with RSConnectClient(connect_server) as client: # bundle_id not provided so grab the latest if not guid_with_bundle.bundle_id: content = client.get_content(guid_with_bundle.guid) @@ -309,7 +309,7 @@ def get_content(connect_server, guid): :param guid: a single guid as a string or list of guids. :return: a list of content items. """ - with RSConnectClient(connect_server, timeout=120) as client: + with RSConnectClient(connect_server) as client: if isinstance(guid, str): result = [client.get_content(guid)] else: @@ -320,7 +320,7 @@ def get_content(connect_server, guid): def search_content( connect_server, published, unpublished, content_type, r_version, py_version, title_contains, order_by ): - with RSConnectClient(connect_server, timeout=120) as client: + with RSConnectClient(connect_server) as client: result = client.search_content() result = _apply_content_filters( result, published, unpublished, content_type, r_version, py_version, title_contains diff --git a/rsconnect/api.py b/rsconnect/api.py index 22c868e0..9a177119 100644 --- a/rsconnect/api.py +++ b/rsconnect/api.py @@ -28,6 +28,7 @@ from .metadata import ServerStore, AppStore from .exception import RSConnectException from .bundle import _default_title, fake_module_file_from_directory +from .timeouts import get_timeout class AbstractRemoteServer: @@ -127,7 +128,7 @@ def __init__(self, url: str): class RSConnectClient(HTTPServer): - def __init__(self, server: RSConnectServer, cookies=None, timeout=30): + def __init__(self, server: RSConnectServer, cookies=None): if cookies is None: cookies = server.cookie_jar super().__init__( @@ -135,7 +136,6 @@ def __init__(self, server: RSConnectServer, cookies=None, timeout=30): server.insecure, server.ca_data, cookies, - timeout, ) self._server = server @@ -282,7 +282,7 @@ def get_content(self, content_guid): return results def wait_for_task( - self, task_id, log_callback, abort_func=lambda: False, timeout=None, poll_wait=0.5, raise_on_error=True + self, task_id, log_callback, abort_func=lambda: False, timeout=get_timeout(), poll_wait=0.5, raise_on_error=True ): last_status = None @@ -387,7 +387,7 @@ def __init__( token=token, secret=secret, ) - self.setup_client(cookies, timeout) + self.setup_client(cookies) @classmethod def fromConnectServer(cls, connect_server, **kwargs): @@ -485,11 +485,11 @@ def setup_remote_server( else: raise RSConnectException("Unable to infer Connect server type and setup server.") - def setup_client(self, cookies=None, timeout=30, **kwargs): + def setup_client(self, cookies=None, **kwargs): if isinstance(self.remote_server, RSConnectServer): - self.client = RSConnectClient(self.remote_server, cookies, timeout) + self.client = RSConnectClient(self.remote_server, cookies) elif isinstance(self.remote_server, PositServer): - self.client = PositClient(self.remote_server, timeout) + self.client = PositClient(self.remote_server) else: raise RSConnectException("Unable to infer Connect client.") @@ -678,7 +678,7 @@ def check_server_capabilities(self, capability_functions): def upload_rstudio_bundle(self, prepare_deploy_result, bundle_size: int, contents): upload_url = prepare_deploy_result.presigned_url parsed_upload_url = urlparse(upload_url) - with S3Client("{}://{}".format(parsed_upload_url.scheme, parsed_upload_url.netloc), timeout=120) as s3_client: + with S3Client("{}://{}".format(parsed_upload_url.scheme, parsed_upload_url.netloc)) as s3_client: upload_result = s3_client.upload( "{}?{}".format(parsed_upload_url.path, parsed_upload_url.query), prepare_deploy_result.presigned_checksum, @@ -1028,14 +1028,14 @@ class PositClient(HTTPServer): _TERMINAL_STATUSES = {"success", "failed", "error"} - def __init__(self, rstudio_server: PositServer, timeout: int = 30): + def __init__(self, rstudio_server: PositServer): self._token = rstudio_server.token try: self._key = base64.b64decode(rstudio_server.secret) except binascii.Error as e: raise RSConnectException("Invalid secret.") from e self._server = rstudio_server - super().__init__(rstudio_server.url, timeout=timeout) + super().__init__(rstudio_server.url) def _get_canonical_request(self, method, path, timestamp, content_hash): return "\n".join([method, path, timestamp, content_hash]) @@ -1121,7 +1121,7 @@ def get_task(self, task_id): def get_current_user(self): return self.get("/v1/users/me") - def wait_until_task_is_successful(self, task_id, timeout=180): + def wait_until_task_is_successful(self, task_id, timeout=get_timeout()): print() print("Waiting for task: {}".format(task_id)) start_time = time.time() @@ -1372,7 +1372,7 @@ def emit_task_log( task_id, log_callback, abort_func=lambda: False, - timeout=None, + timeout=get_timeout(), poll_wait=0.5, raise_on_error=True, ): diff --git a/rsconnect/http_support.py b/rsconnect/http_support.py index b863c720..ae9ed5cf 100644 --- a/rsconnect/http_support.py +++ b/rsconnect/http_support.py @@ -13,11 +13,13 @@ from six.moves.urllib_parse import urlparse, urlencode, urljoin import base64 +from .timeouts import get_timeout + _user_agent = "rsconnect-python/%s" % VERSION # noinspection PyUnusedLocal,PyUnresolvedReferences -def _create_plain_connection(host_name, port, disable_tls_check, ca_data, timeout): +def _create_plain_connection(host_name, port, disable_tls_check, ca_data): """ This function is used to create a plain HTTP connection. Note that the 3rd and 4th parameters are ignored; they are present to make the signature match the companion @@ -27,9 +29,10 @@ def _create_plain_connection(host_name, port, disable_tls_check, ca_data, timeou :param port: the port to connect to. :param disable_tls_check: notes whether TLS verification should be disabled (ignored). :param ca_data: any certificate authority information to use (ignored). - :param timeout: the timeout value to use for socket operations. :return: a plain HTTP connection. """ + timeout = get_timeout() + logger.debug(f"The HTTPConnection timeout is set to '{timeout}' seconds") return http.HTTPConnection(host_name, port=(port or http.HTTP_PORT), timeout=timeout) @@ -59,7 +62,7 @@ def _get_proxy_headers(*args, **kwargs): # noinspection PyUnresolvedReferences -def _create_ssl_connection(host_name, port, disable_tls_check, ca_data, timeout): +def _create_ssl_connection(host_name, port, disable_tls_check, ca_data): """ This function is used to create a TLS encrypted HTTP connection (SSL). @@ -74,6 +77,8 @@ def _create_ssl_connection(host_name, port, disable_tls_check, ca_data, timeout) raise ValueError("Cannot both disable TLS checking and provide a custom certificate") _, _, proxyHost, proxyPort = _get_proxy() headers = _get_proxy_headers() + timeout = get_timeout() + logger.debug(f"The HTTPSConnection timeout is set to '{timeout}' seconds") if ca_data is not None: return http.HTTPSConnection( host_name, @@ -162,7 +167,7 @@ class HTTPServer(object): server. """ - def __init__(self, url, disable_tls_check=False, ca_data=None, cookies=None, timeout=30): + def __init__(self, url, disable_tls_check=False, ca_data=None, cookies=None): """ Constructs an HTTPServer object. @@ -174,7 +179,6 @@ def __init__(self, url, disable_tls_check=False, ca_data=None, cookies=None, tim certificates. :param cookies: an optional cookie jar. Must be of type `CookieJar` defined in this same file (i.e., not the one Python provides). - :param timeout: the timeout value to use for socket operations. """ self._url = urlparse(url) @@ -184,7 +188,6 @@ def __init__(self, url, disable_tls_check=False, ca_data=None, cookies=None, tim self._disable_tls_check = disable_tls_check self._ca_data = ca_data self._cookies = cookies if cookies is not None else CookieJar() - self._timeout = timeout self._headers = {"User-Agent": _user_agent} self._conn = None self._proxy_headers = _get_proxy_headers() @@ -216,7 +219,6 @@ def __enter__(self): self._url.port, self._disable_tls_check, self._ca_data, - self._timeout, ) return self diff --git a/rsconnect/timeouts.py b/rsconnect/timeouts.py new file mode 100644 index 00000000..e86b96f2 --- /dev/null +++ b/rsconnect/timeouts.py @@ -0,0 +1,41 @@ +import os +from typing import Union + +from rsconnect.exception import RSConnectException + +_CONNECT_REQUEST_TIMEOUT_KEY = "CONNECT_REQUEST_TIMEOUT" +_CONNECT_REQUEST_TIMEOUT_DEFAULT_VALUE = "300" + + +def get_timeout() -> int: + """Gets the timeout from the CONNECT_REQUEST_TIMEOUT env variable. + + The timeout value is intended to be interpreted in seconds. A value of 60 is equal to sixty seconds, or one minute. + + If CONNECT_REQUEST_TIMEOUT is unset, a default value of 300 is used. + + If CONNECT_REQUEST_TIMEOUT is set to a value less than 0, an `RSConnectException` is raised. + + A CONNECT_REQUEST_TIMEOUT set to 0 is logically equivalent to no timeout. + + The primary intent for this method is for usage with the `http` module. Specifically, for setting the timeout + parameter with an `http.client.HTTPConnection` or `http.client.HTTPSConnection`. + + :raises: `RSConnectException` if CONNECT_REQUEST_TIMEOUT is not a natural number. + :return: the timeout value + """ + timeout: Union[int, str] = os.environ.get(_CONNECT_REQUEST_TIMEOUT_KEY, _CONNECT_REQUEST_TIMEOUT_DEFAULT_VALUE) + + try: + timeout = int(timeout) + except ValueError: + raise RSConnectException( + f"'CONNECT_REQUEST_TIMEOUT' is set to '{timeout}'. The value must be a natural number." + ) + + if timeout < 0: + raise RSConnectException( + f"'CONNECT_REQUEST_TIMEOUT' is set to '{timeout}'. The value must be a natural number." + ) + + return timeout diff --git a/tests/test_http_support.py b/tests/test_http_support.py index 551a18ce..f7c2be1d 100644 --- a/tests/test_http_support.py +++ b/tests/test_http_support.py @@ -19,7 +19,7 @@ def test_connection_factory_map(self): def test_create_ssl_checks(self): with self.assertRaises(ValueError): - _create_ssl_connection(None, None, True, "fake", 10) + _create_ssl_connection(None, None, True, "fake") def test_append_to_path(self): self.assertEqual(append_to_path("path/", "/sub"), "path/sub") diff --git a/tests/test_timeouts.py b/tests/test_timeouts.py new file mode 100644 index 00000000..da231deb --- /dev/null +++ b/tests/test_timeouts.py @@ -0,0 +1,33 @@ +import os + +from unittest import TestCase +from unittest.mock import patch + +from rsconnect.exception import RSConnectException +from rsconnect.timeouts import get_timeout + + +class GetTimeoutTestCase(TestCase): + def test_get_default_timeout(self): + timeout = get_timeout() + self.assertEqual(300, timeout) + + def test_get_valid_timeout_from_environment(self): + with patch.dict(os.environ, {"CONNECT_REQUEST_TIMEOUT": "24"}): + timeout = get_timeout() + self.assertEqual(24, timeout) + + def test_get_zero_timeout_from_environment(self): + with patch.dict(os.environ, {"CONNECT_REQUEST_TIMEOUT": "0"}): + timeout = get_timeout() + self.assertEqual(0, timeout) + + def test_get_invalid_timeout_from_environment(self): + with patch.dict(os.environ, {"CONNECT_REQUEST_TIMEOUT": "foobar"}): + with self.assertRaises(RSConnectException): + get_timeout() + + def test_get_negative_timeout_from_environment(self): + with patch.dict(os.environ, {"CONNECT_REQUEST_TIMEOUT": "-24"}): + with self.assertRaises(RSConnectException): + get_timeout() From f1054aac604abc23c906aca813db945763f12f4c Mon Sep 17 00:00:00 2001 From: tdstein Date: Fri, 10 Feb 2023 10:11:55 -0500 Subject: [PATCH 02/16] Adds notes to CHANGELOG for CONNECT_REQUEST_TIMEOUT. --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 238ea2c2..a640b00a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## Unreleased + +### Added + +- The `CONNECT_REQUEST_TIMEOUT` environment variable, which configures the request timeout for all blocking HTTP and HTTPS operations. This value translates into seconds (e.g., `CONNECT_REQUEST_TIMEOUT=60` is equivalent to 60 seconds.) By default, this value is 300. + ## [1.14.1] - 2023-02-09 ### Fixed From 796c901fb58068c2d6cf3f974d6f7c72049c28b9 Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Wed, 8 Mar 2023 11:11:11 -0500 Subject: [PATCH 03/16] add absolute path file list and use it for voila --- rsconnect/bundle.py | 50 +++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 46 insertions(+), 4 deletions(-) diff --git a/rsconnect/bundle.py b/rsconnect/bundle.py index 235f4fc0..e3ce9f49 100644 --- a/rsconnect/bundle.py +++ b/rsconnect/bundle.py @@ -890,7 +890,7 @@ def create_file_list( file_set = set(extra_files) # type: typing.Set[str] if isfile(path): - file_set.add(Path(path).name) + file_set.add(path) return sorted(file_set) for cur_dir, sub_dirs, files in os.walk(path): @@ -911,6 +911,48 @@ def create_file_list( return sorted(file_set) +def create_abspath_list( + path: str, + extra_files: typing.List[str] = None, + excludes: typing.List[str] = None, +) -> typing.List[str]: + """ + Builds a absolute path list of files under the given path that should be included + in a manifest or bundle. + + :param path: a file, or a directory to walk for files. + :param extra_files: a sequence of any extra files to include in the bundle. + :param excludes: a sequence of glob patterns that will exclude matched files. + :return: the list of relevant files, relative to the given directory. + """ + extra_files = extra_files or [] + excludes = excludes if excludes else [] + glob_set = create_glob_set(path, excludes) + exclude_paths = {Path(p) for p in excludes} + file_set = set(extra_files) # type: typing.Set[str] + + if isfile(path): + file_set.add(abspath(path)) + return sorted(file_set) + + for cur_dir, sub_dirs, files in os.walk(path): + if Path(cur_dir) in exclude_paths: + continue + if any(parent in exclude_paths for parent in Path(cur_dir).parents): + continue + for file in files: + abs_path = os.path.join(cur_dir, file) + rel_path = relpath(abs_path, path) + + if Path(abs_path) in exclude_paths: + continue + if keep_manifest_specified_file(rel_path, exclude_paths | directories_to_ignore) and ( + rel_path in extra_files or not glob_set.matches(abs_path) + ): + file_set.add(abspath(abs_path)) + return sorted(file_set) + + def infer_entrypoint(path, mimetype): candidates = infer_entrypoint_candidates(path, mimetype) return candidates.pop() if len(candidates) == 1 else None @@ -1646,9 +1688,9 @@ def create_voila_manifest( manifest.add_to_buffer(join(deploy_dir, environment.filename), environment.contents) - file_list = create_file_list(path, extra_files, excludes) - for rel_path in file_list: - manifest.add_relative_path(rel_path) + file_list = create_abspath_list(path, extra_files, excludes) + for abs_path in file_list: + manifest.add_file(abs_path) return manifest From f2f01e2523ab7dadc616b3d00f26aa62fcb888e6 Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Wed, 8 Mar 2023 12:36:53 -0500 Subject: [PATCH 04/16] return list instead of string when default found --- rsconnect/bundle.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rsconnect/bundle.py b/rsconnect/bundle.py index e3ce9f49..2ce5065b 100644 --- a/rsconnect/bundle.py +++ b/rsconnect/bundle.py @@ -977,7 +977,7 @@ def infer_entrypoint_candidates(path, mimetype) -> List: continue mimetype_filelist[guess_type(file)[0]].append(rel_path) if file in default_mimetype_entrypoints[mimetype]: - return file + return [file] return mimetype_filelist[mimetype] or [] From 83459991ed270f5ba97d7ac384025bc80b3c8e62 Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Wed, 8 Mar 2023 18:58:44 -0500 Subject: [PATCH 05/16] add deploy html unittest files --- tests/testdata/html_tests/multi_file_index/index.html | 9 +++++++++ tests/testdata/html_tests/multi_file_index/main.html | 9 +++++++++ tests/testdata/html_tests/multi_file_nonindex/a.html | 9 +++++++++ tests/testdata/html_tests/multi_file_nonindex/b.html | 9 +++++++++ tests/testdata/html_tests/single_file_index/index.html | 9 +++++++++ tests/testdata/html_tests/single_file_index/test1.txt | 1 + .../single_file_index/test_folder1/testfoldertext1.txt | 1 + tests/testdata/html_tests/single_file_nonindex/main.html | 9 +++++++++ tests/testdata/html_tests/single_file_nonindex/test1.txt | 1 + .../test_folder1/testfoldertext1.txt | 1 + 10 files changed, 58 insertions(+) create mode 100644 tests/testdata/html_tests/multi_file_index/index.html create mode 100644 tests/testdata/html_tests/multi_file_index/main.html create mode 100644 tests/testdata/html_tests/multi_file_nonindex/a.html create mode 100644 tests/testdata/html_tests/multi_file_nonindex/b.html create mode 100644 tests/testdata/html_tests/single_file_index/index.html create mode 100644 tests/testdata/html_tests/single_file_index/test1.txt create mode 100644 tests/testdata/html_tests/single_file_index/test_folder1/testfoldertext1.txt create mode 100644 tests/testdata/html_tests/single_file_nonindex/main.html create mode 100644 tests/testdata/html_tests/single_file_nonindex/test1.txt create mode 100644 tests/testdata/html_tests/single_file_nonindex/test_folder1/testfoldertext1.txt diff --git a/tests/testdata/html_tests/multi_file_index/index.html b/tests/testdata/html_tests/multi_file_index/index.html new file mode 100644 index 00000000..2e1a4c77 --- /dev/null +++ b/tests/testdata/html_tests/multi_file_index/index.html @@ -0,0 +1,9 @@ + + + + +

This file is intentionally left blank.

+

This file is intentionally left blank.

+ + + diff --git a/tests/testdata/html_tests/multi_file_index/main.html b/tests/testdata/html_tests/multi_file_index/main.html new file mode 100644 index 00000000..2e1a4c77 --- /dev/null +++ b/tests/testdata/html_tests/multi_file_index/main.html @@ -0,0 +1,9 @@ + + + + +

This file is intentionally left blank.

+

This file is intentionally left blank.

+ + + diff --git a/tests/testdata/html_tests/multi_file_nonindex/a.html b/tests/testdata/html_tests/multi_file_nonindex/a.html new file mode 100644 index 00000000..2e1a4c77 --- /dev/null +++ b/tests/testdata/html_tests/multi_file_nonindex/a.html @@ -0,0 +1,9 @@ + + + + +

This file is intentionally left blank.

+

This file is intentionally left blank.

+ + + diff --git a/tests/testdata/html_tests/multi_file_nonindex/b.html b/tests/testdata/html_tests/multi_file_nonindex/b.html new file mode 100644 index 00000000..2e1a4c77 --- /dev/null +++ b/tests/testdata/html_tests/multi_file_nonindex/b.html @@ -0,0 +1,9 @@ + + + + +

This file is intentionally left blank.

+

This file is intentionally left blank.

+ + + diff --git a/tests/testdata/html_tests/single_file_index/index.html b/tests/testdata/html_tests/single_file_index/index.html new file mode 100644 index 00000000..2e1a4c77 --- /dev/null +++ b/tests/testdata/html_tests/single_file_index/index.html @@ -0,0 +1,9 @@ + + + + +

This file is intentionally left blank.

+

This file is intentionally left blank.

+ + + diff --git a/tests/testdata/html_tests/single_file_index/test1.txt b/tests/testdata/html_tests/single_file_index/test1.txt new file mode 100644 index 00000000..a5bce3fd --- /dev/null +++ b/tests/testdata/html_tests/single_file_index/test1.txt @@ -0,0 +1 @@ +test1 diff --git a/tests/testdata/html_tests/single_file_index/test_folder1/testfoldertext1.txt b/tests/testdata/html_tests/single_file_index/test_folder1/testfoldertext1.txt new file mode 100644 index 00000000..e8ca6258 --- /dev/null +++ b/tests/testdata/html_tests/single_file_index/test_folder1/testfoldertext1.txt @@ -0,0 +1 @@ +testfoldertext1 diff --git a/tests/testdata/html_tests/single_file_nonindex/main.html b/tests/testdata/html_tests/single_file_nonindex/main.html new file mode 100644 index 00000000..2e1a4c77 --- /dev/null +++ b/tests/testdata/html_tests/single_file_nonindex/main.html @@ -0,0 +1,9 @@ + + + + +

This file is intentionally left blank.

+

This file is intentionally left blank.

+ + + diff --git a/tests/testdata/html_tests/single_file_nonindex/test1.txt b/tests/testdata/html_tests/single_file_nonindex/test1.txt new file mode 100644 index 00000000..a5bce3fd --- /dev/null +++ b/tests/testdata/html_tests/single_file_nonindex/test1.txt @@ -0,0 +1 @@ +test1 diff --git a/tests/testdata/html_tests/single_file_nonindex/test_folder1/testfoldertext1.txt b/tests/testdata/html_tests/single_file_nonindex/test_folder1/testfoldertext1.txt new file mode 100644 index 00000000..e8ca6258 --- /dev/null +++ b/tests/testdata/html_tests/single_file_nonindex/test_folder1/testfoldertext1.txt @@ -0,0 +1 @@ +testfoldertext1 From 00f818b1d84ff4102d8e18e378271f2fe1848a56 Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Wed, 8 Mar 2023 19:17:50 -0500 Subject: [PATCH 06/16] deprecate legacy deploy_html functions - make_html_bundle - make_html_bundle_content --- rsconnect/bundle.py | 83 --------------------------------------------- 1 file changed, 83 deletions(-) diff --git a/rsconnect/bundle.py b/rsconnect/bundle.py index 2ce5065b..9a5b2781 100644 --- a/rsconnect/bundle.py +++ b/rsconnect/bundle.py @@ -817,57 +817,6 @@ def make_api_manifest( return manifest, relevant_files -def make_html_bundle_content( - path: str, - entrypoint: str, - extra_files: typing.List[str], - excludes: typing.List[str], - image: str = None, -) -> typing.Tuple[typing.Dict[str, typing.Any], typing.List[str]]: - - """ - Makes a manifest for static html deployment. - - :param path: the file, or the directory containing the files to deploy. - :param entry_point: the main entry point for the API. - :param extra_files: a sequence of any extra files to include in the bundle. - :param excludes: a sequence of glob patterns that will exclude matched files. - :param image: the optional docker image to be specified for off-host execution. Default = None. - :return: the manifest and a list of the files involved. - """ - extra_files = list(extra_files) if extra_files else [] - entrypoint = entrypoint or infer_entrypoint(path=path, mimetype="text/html") - if not entrypoint: - raise RSConnectException("Unable to find a valid html entry point.") - - if path.startswith(os.curdir): - path = relpath(path) - if entrypoint.startswith(os.curdir): - entrypoint = relpath(entrypoint) - extra_files = [relpath(f) if isfile(f) and f.startswith(os.curdir) else f for f in extra_files] - - if is_environment_dir(path): - excludes = list(excludes or []) + ["bin/", "lib/"] - - extra_files = extra_files or [] - skip = ["manifest.json"] - extra_files = sorted(set(extra_files) - set(skip)) - - # Don't include these top-level files. - excludes = list(excludes) if excludes else [] - excludes.append("manifest.json") - if not isfile(path): - excludes.extend(list_environment_dirs(path)) - - relevant_files = create_file_list(path, extra_files, excludes) - manifest = make_html_manifest(entrypoint, image) - - for rel_path in relevant_files: - manifest_add_file(manifest, rel_path, path) - - return manifest, relevant_files - - def create_file_list( path: str, extra_files: typing.List[str] = None, @@ -981,38 +930,6 @@ def infer_entrypoint_candidates(path, mimetype) -> List: return mimetype_filelist[mimetype] or [] -def make_html_bundle( - path: str, - entry_point: str, - extra_files: typing.List[str], - excludes: typing.List[str], - image: str = None, -) -> typing.IO[bytes]: - """ - Create an html bundle, given a path and a manifest. - - :param path: the file, or the directory containing the files to deploy. - :param entry_point: the main entry point for the API. - :param extra_files: a sequence of any extra files to include in the bundle. - :param excludes: a sequence of glob patterns that will exclude matched files. - :param image: the optional docker image to be specified for off-host execution. Default = None. - :return: a file-like object containing the bundle tarball. - """ - manifest, relevant_files = make_html_bundle_content(path, entry_point, extra_files, excludes, image) - bundle_file = tempfile.TemporaryFile(prefix="rsc_bundle") - - with tarfile.open(mode="w:gz", fileobj=bundle_file) as bundle: - bundle_add_buffer(bundle, "manifest.json", json.dumps(manifest, indent=2)) - - for rel_path in relevant_files: - bundle_add_file(bundle, rel_path, path) - - # rewind file pointer - bundle_file.seek(0) - - return bundle_file - - def guess_deploy_dir(path, entrypoint): abs_path = abspath(path) if path else None abs_entrypoint = abspath(entrypoint) if entrypoint else None From 5330a32015167e33aa6877cb23008a1a19f368e1 Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Wed, 8 Mar 2023 19:23:40 -0500 Subject: [PATCH 07/16] update infer_entrypoint_candidates to return absolute paths --- rsconnect/bundle.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/rsconnect/bundle.py b/rsconnect/bundle.py index 9a5b2781..224f78ab 100644 --- a/rsconnect/bundle.py +++ b/rsconnect/bundle.py @@ -921,12 +921,12 @@ def infer_entrypoint_candidates(path, mimetype) -> List: mimetype_filelist = defaultdict(list) for file in os.listdir(path): - rel_path = os.path.join(path, file) - if not isfile(rel_path): + abs_path = os.path.join(path, file) + if not isfile(abs_path): continue - mimetype_filelist[guess_type(file)[0]].append(rel_path) + mimetype_filelist[guess_type(file)[0]].append(abs_path) if file in default_mimetype_entrypoints[mimetype]: - return [file] + return [abs_path] return mimetype_filelist[mimetype] or [] From d203ef7ba10124658011cdbee77c593600e65eb1 Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Wed, 8 Mar 2023 19:25:39 -0500 Subject: [PATCH 08/16] refactor deploy_html with Manifest and Bundle --- rsconnect/bundle.py | 101 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 101 insertions(+) diff --git a/rsconnect/bundle.py b/rsconnect/bundle.py index 224f78ab..bf05a6da 100644 --- a/rsconnect/bundle.py +++ b/rsconnect/bundle.py @@ -817,6 +817,107 @@ def make_api_manifest( return manifest, relevant_files +def create_html_manifest( + path: str, + entrypoint: str, + app_mode: AppMode = AppModes.STATIC, + extra_files: typing.List[str] = None, + excludes: typing.List[str] = None, + image: str = None, + **kwargs +) -> Manifest: + """ + Creates and writes a manifest.json file for the given path. + + :param path: the file, or the directory containing the files to deploy. + :param entry_point: the main entry point for the API. + :param environment: the Python environment to start with. This should be what's + returned by the inspect_environment() function. + :param app_mode: the application mode to assume. If this is None, the extension + portion of the entry point file name will be used to derive one. Previous default = None. + :param extra_files: any extra files that should be included in the manifest. Previous default = None. + :param excludes: a sequence of glob patterns that will exclude matched files. + :param force_generate: bool indicating whether to force generate manifest and related environment files. + :param image: the optional docker image to be specified for off-host execution. Default = None. + :return: the manifest data structure. + """ + if not path: + raise RSConnectException("A valid path must be provided.") + extra_files = list(extra_files) if extra_files else [] + entrypoint_candidates = infer_entrypoint_candidates(path=abspath(path), mimetype="text/html") + + deploy_dir = guess_deploy_dir(path, entrypoint) + if len(entrypoint_candidates) <= 0: + if entrypoint is None: + raise RSConnectException("No valid entrypoint found.") + entrypoint = abs_entrypoint(path, entrypoint) + elif len(entrypoint_candidates) == 1: + if entrypoint: + entrypoint = abs_entrypoint(path, entrypoint) + else: + entrypoint = entrypoint_candidates[0] + else: # len(entrypoint_candidates) > 1: + if entrypoint is None: + raise RSConnectException("No valid entrypoint found.") + entrypoint = abs_entrypoint(path, entrypoint) + + extra_files = validate_extra_files(deploy_dir, extra_files) + excludes = list(excludes) if excludes else [] + excludes.extend(["manifest.json"]) + excludes.extend(list_environment_dirs(deploy_dir)) + + manifest = Manifest(app_mode=AppModes.STATIC, entrypoint=entrypoint, image=image) + manifest.deploy_dir = deploy_dir + + file_list = create_abspath_list(path, extra_files, excludes) + for abs_path in file_list: + manifest.add_file(abs_path) + + return manifest + + +def make_html_bundle( + path: str, + entrypoint: str, + extra_files: typing.List[str], + excludes: typing.List[str], + image: str = None, +) -> typing.IO[bytes]: + """ + Create an voila bundle, given a path and/or entrypoint. + + The bundle contains a manifest.json file created for the given notebook entrypoint file. + If the related environment file (requirements.txt) doesn't + exist (or force_generate is set to True), the environment file will also be written. + + :param path: the file, or the directory containing the files to deploy. + :param entry_point: the main entry point. + :param extra_files: a sequence of any extra files to include in the bundle. + :param excludes: a sequence of glob patterns that will exclude matched files. + :param force_generate: bool indicating whether to force generate manifest and related environment files. + :param image: the optional docker image to be specified for off-host execution. Default = None. + :return: a file-like object containing the bundle tarball. + """ + + manifest = create_html_manifest(**locals()) + if manifest.data.get("files") is None: + raise RSConnectException("No valid files were found for the manifest.") + + bundle = Bundle() + for f in manifest.data["files"]: + if f in manifest.buffer: + continue + bundle.add_file(f) + for k, v in manifest.flattened_buffer.items(): + bundle.add_to_buffer(k, v) + + manifest_flattened_copy_data = manifest.flattened_copy.data + bundle.add_to_buffer("manifest.json", json.dumps(manifest_flattened_copy_data, indent=2)) + bundle.deploy_dir = manifest.deploy_dir + + return bundle.to_file() + + def create_file_list( path: str, extra_files: typing.List[str] = None, From b4e7fd9266fe10c8dc78d0f4dcdaeb0c894f2994 Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Wed, 8 Mar 2023 19:32:03 -0500 Subject: [PATCH 09/16] add unit tests for deploy_html manifest --- tests/test_bundle.py | 130 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 130 insertions(+) diff --git a/tests/test_bundle.py b/tests/test_bundle.py index 834b97e8..0a8601cd 100644 --- a/tests/test_bundle.py +++ b/tests/test_bundle.py @@ -14,6 +14,7 @@ _default_title, _default_title_from_manifest, _validate_title, + create_html_manifest, get_python_env_info, inspect_environment, list_files, @@ -1341,3 +1342,132 @@ def test_make_voila_bundle_2( reqs = tar.extractfile("requirements.txt").read() assert reqs == b"numpy\nipywidgets\nbqplot\n" assert ans == json.loads(tar.extractfile("manifest.json").read().decode("utf-8")) + + +single_file_index_dir = os.path.join(cur_dir, "./testdata/html_tests/single_file_index") +single_file_index_file = os.path.join(cur_dir, "./testdata/html_tests/single_file_index/index.html") +single_file_nonindex_dir = os.path.join(cur_dir, "./testdata/html_tests/single_file_nonindex") +multi_file_index_dir = os.path.join(cur_dir, "./testdata/html_tests/multi_file_index") +multi_file_index_file = os.path.join(cur_dir, "./testdata/html_tests/multi_file_index/index.html") +multi_file_nonindex_dir = os.path.join(cur_dir, "./testdata/html_tests/multi_file_nonindex") +multi_file_nonindex_file = os.path.join(cur_dir, "./testdata/html_tests/multi_file_nonindex/b.html") + + +def test_create_html_manifest(): + + with pytest.raises(RSConnectException) as _: + _, _ = create_html_manifest( + None, + None, + extra_files=None, + excludes=None, + image=None, + ) + with pytest.raises(RSConnectException) as _: + _, _ = create_html_manifest( + None, + single_file_index_file, + extra_files=None, + excludes=None, + image=None, + ) + with pytest.raises(RSConnectException) as _: + _, _ = create_html_manifest( + multi_file_nonindex_dir, + None, + extra_files=None, + excludes=None, + image=None, + ) + single_file_index_file_ans = { + "version": 1, + "metadata": {"appmode": "static", "entrypoint": "index.html"}, + "files": {"index.html": {"checksum": "c14bd63e50295f94b761ffe9d41e3742"}}, + } + manifest = create_html_manifest( + single_file_index_file, + None, + ) + assert single_file_index_file_ans == json.loads(manifest.flattened_copy.json) + + single_file_index_dir_ans = { + "version": 1, + "metadata": {"appmode": "static", "entrypoint": "index.html"}, + "files": { + "index.html": {"checksum": "c14bd63e50295f94b761ffe9d41e3742"}, + "test1.txt": {"checksum": "3e7705498e8be60520841409ebc69bc1"}, + "test_folder1/testfoldertext1.txt": {"checksum": "0a576fd324b6985bac6aa934131d2f5c"}, + }, + } + + manifest = create_html_manifest( + single_file_index_dir, + None, + ) + assert single_file_index_dir_ans == json.loads(manifest.flattened_copy.json) + + manifest = create_html_manifest( + single_file_index_dir, + single_file_index_file, + ) + assert single_file_index_dir_ans == json.loads(manifest.flattened_copy.json) + + multi_file_index_file_ans = { + "version": 1, + "metadata": {"appmode": "static", "entrypoint": "index.html"}, + "files": {"index.html": {"checksum": "c14bd63e50295f94b761ffe9d41e3742"}}, + } + + manifest = create_html_manifest( + multi_file_index_file, + None, + ) + assert multi_file_index_file_ans == json.loads(manifest.flattened_copy.json) + + multi_file_index_dir_ans = { + "version": 1, + "metadata": {"appmode": "static", "entrypoint": "index.html"}, + "files": { + "index.html": {"checksum": "c14bd63e50295f94b761ffe9d41e3742"}, + "main.html": {"checksum": "c14bd63e50295f94b761ffe9d41e3742"}, + }, + } + + manifest = create_html_manifest( + multi_file_index_dir, + None, + ) + assert multi_file_index_dir_ans == json.loads(manifest.flattened_copy.json) + + manifest = create_html_manifest( + multi_file_index_dir, + multi_file_index_file, + ) + assert multi_file_index_dir_ans == json.loads(manifest.flattened_copy.json) + + multi_file_nonindex_file_ans = { + "version": 1, + "metadata": {"appmode": "static", "entrypoint": "b.html"}, + "files": {"b.html": {"checksum": "c14bd63e50295f94b761ffe9d41e3742"}}, + } + + manifest = create_html_manifest( + multi_file_nonindex_file, + None, + ) + assert multi_file_nonindex_file_ans == json.loads(manifest.flattened_copy.json) + + multi_file_nonindex_dir_and_file_ans = { + "version": 1, + "metadata": {"appmode": "static", "entrypoint": "b.html"}, + "files": { + "a.html": {"checksum": "c14bd63e50295f94b761ffe9d41e3742"}, + "b.html": {"checksum": "c14bd63e50295f94b761ffe9d41e3742"}, + }, + } + + manifest = create_html_manifest( + multi_file_nonindex_dir, + multi_file_nonindex_file, + ) + assert multi_file_nonindex_dir_and_file_ans == json.loads(manifest.flattened_copy.json) From c7f53f9ba48cac6e8b2dd451b2db95d18593ef2b Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Thu, 9 Mar 2023 12:37:03 -0500 Subject: [PATCH 10/16] add primary_html to Manifest --- rsconnect/bundle.py | 25 ++++++++++++++++++++++++- tests/test_bundle.py | 12 ++++++------ 2 files changed, 30 insertions(+), 7 deletions(-) diff --git a/rsconnect/bundle.py b/rsconnect/bundle.py index bf05a6da..4ab4594d 100644 --- a/rsconnect/bundle.py +++ b/rsconnect/bundle.py @@ -68,6 +68,7 @@ def __init__(self, *args, **kwargs) -> None: quarto_inspection = kwargs.get("quarto_inspection") environment = kwargs.get("environment") image = kwargs.get("image") + primary_html = kwargs.get("primary_html") self.data["version"] = version if version else 1 if environment: @@ -82,6 +83,8 @@ def __init__(self, *args, **kwargs) -> None: "appmode": AppModes.UNKNOWN, } ) + if primary_html: + self.data["metadata"]["primary_html"] = primary_html if entrypoint: self.data["metadata"]["entrypoint"] = entrypoint @@ -150,6 +153,18 @@ def entrypoint(self): def entrypoint(self, value): self.data["metadata"]["entrypoint"] = value + @property + def primary_html(self): + if "metadata" not in self.data: + return None + if "primary_html" in self.data["metadata"]: + return self.data["metadata"]["primary_html"] + return None + + @primary_html.setter + def primary_html(self, value): + self.data["metadata"]["primary_html"] = value + def add_file(self, path): self.data["files"][path] = {"checksum": file_checksum(path)} return self @@ -207,6 +222,12 @@ def flattened_entrypoint(self): raise RSConnectException("A valid entrypoint must be provided.") return relpath(self.entrypoint, dirname(self.entrypoint)) + @property + def flattened_primary_html(self): + if self.primary_html is None: + raise RSConnectException("A valid primary_html must be provided.") + return relpath(self.primary_html, dirname(self.primary_html)) + @property def flattened_copy(self): if self.entrypoint is None: @@ -215,6 +236,8 @@ def flattened_copy(self): new_manifest.data["files"] = self.flattened_data new_manifest.buffer = self.flattened_buffer new_manifest.entrypoint = self.flattened_entrypoint + if self.primary_html: + new_manifest.primary_html = self.flattened_primary_html return new_manifest def make_relative_to_deploy_dir(self): @@ -866,7 +889,7 @@ def create_html_manifest( excludes.extend(["manifest.json"]) excludes.extend(list_environment_dirs(deploy_dir)) - manifest = Manifest(app_mode=AppModes.STATIC, entrypoint=entrypoint, image=image) + manifest = Manifest(app_mode=AppModes.STATIC, entrypoint=entrypoint, primary_html=entrypoint, image=image) manifest.deploy_dir = deploy_dir file_list = create_abspath_list(path, extra_files, excludes) diff --git a/tests/test_bundle.py b/tests/test_bundle.py index 0a8601cd..255ef2ce 100644 --- a/tests/test_bundle.py +++ b/tests/test_bundle.py @@ -1381,7 +1381,7 @@ def test_create_html_manifest(): ) single_file_index_file_ans = { "version": 1, - "metadata": {"appmode": "static", "entrypoint": "index.html"}, + "metadata": {"appmode": "static", "primary_html": "index.html", "entrypoint": "index.html"}, "files": {"index.html": {"checksum": "c14bd63e50295f94b761ffe9d41e3742"}}, } manifest = create_html_manifest( @@ -1392,7 +1392,7 @@ def test_create_html_manifest(): single_file_index_dir_ans = { "version": 1, - "metadata": {"appmode": "static", "entrypoint": "index.html"}, + "metadata": {"appmode": "static", "primary_html": "index.html", "entrypoint": "index.html"}, "files": { "index.html": {"checksum": "c14bd63e50295f94b761ffe9d41e3742"}, "test1.txt": {"checksum": "3e7705498e8be60520841409ebc69bc1"}, @@ -1414,7 +1414,7 @@ def test_create_html_manifest(): multi_file_index_file_ans = { "version": 1, - "metadata": {"appmode": "static", "entrypoint": "index.html"}, + "metadata": {"appmode": "static", "primary_html": "index.html", "entrypoint": "index.html"}, "files": {"index.html": {"checksum": "c14bd63e50295f94b761ffe9d41e3742"}}, } @@ -1426,7 +1426,7 @@ def test_create_html_manifest(): multi_file_index_dir_ans = { "version": 1, - "metadata": {"appmode": "static", "entrypoint": "index.html"}, + "metadata": {"appmode": "static", "primary_html": "index.html", "entrypoint": "index.html"}, "files": { "index.html": {"checksum": "c14bd63e50295f94b761ffe9d41e3742"}, "main.html": {"checksum": "c14bd63e50295f94b761ffe9d41e3742"}, @@ -1447,7 +1447,7 @@ def test_create_html_manifest(): multi_file_nonindex_file_ans = { "version": 1, - "metadata": {"appmode": "static", "entrypoint": "b.html"}, + "metadata": {"appmode": "static", "primary_html": "b.html", "entrypoint": "b.html"}, "files": {"b.html": {"checksum": "c14bd63e50295f94b761ffe9d41e3742"}}, } @@ -1459,7 +1459,7 @@ def test_create_html_manifest(): multi_file_nonindex_dir_and_file_ans = { "version": 1, - "metadata": {"appmode": "static", "entrypoint": "b.html"}, + "metadata": {"appmode": "static", "primary_html": "b.html", "entrypoint": "b.html"}, "files": { "a.html": {"checksum": "c14bd63e50295f94b761ffe9d41e3742"}, "b.html": {"checksum": "c14bd63e50295f94b761ffe9d41e3742"}, From e3bf8b2ba8b7a5a7254aa6ee7c2bd90beab6126c Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Thu, 9 Mar 2023 17:02:00 -0500 Subject: [PATCH 11/16] check whether provided path and entrypoint exist --- rsconnect/bundle.py | 4 ++++ tests/test_bundle.py | 8 ++++++++ 2 files changed, 12 insertions(+) diff --git a/rsconnect/bundle.py b/rsconnect/bundle.py index 4ab4594d..28933598 100644 --- a/rsconnect/bundle.py +++ b/rsconnect/bundle.py @@ -1055,6 +1055,10 @@ def infer_entrypoint_candidates(path, mimetype) -> List: def guess_deploy_dir(path, entrypoint): + if path and not exists(path): + raise RSConnectException(f"Path {path} does not exist.") + if entrypoint and not exists(entrypoint): + raise RSConnectException(f"Entrypoint {entrypoint} does not exist.") abs_path = abspath(path) if path else None abs_entrypoint = abspath(entrypoint) if entrypoint else None if not path and not entrypoint: diff --git a/tests/test_bundle.py b/tests/test_bundle.py index 255ef2ce..56ce740f 100644 --- a/tests/test_bundle.py +++ b/tests/test_bundle.py @@ -814,6 +814,8 @@ def test_is_not_executable(self): dashboard_dir = os.path.join(cur_dir, "./testdata/voila/dashboard/") dashboard_ipynb = os.path.join(dashboard_dir, "dashboard.ipynb") multivoila_dir = os.path.join(cur_dir, "./testdata/voila/multi-voila/") +nonexistent_dir = os.path.join(cur_dir, "./testdata/nonexistent/") +nonexistent_file = os.path.join(cur_dir, "nonexistent.txt") class Test_guess_deploy_dir(TestCase): @@ -824,6 +826,12 @@ def test_guess_deploy_dir(self): guess_deploy_dir(None, bqplot_dir) with self.assertRaises(RSConnectException): guess_deploy_dir(bqplot_dir, bqplot_dir) + with self.assertRaises(RSConnectException): + guess_deploy_dir(nonexistent_dir, None) + with self.assertRaises(RSConnectException): + guess_deploy_dir(None, nonexistent_file) + with self.assertRaises(RSConnectException): + guess_deploy_dir(nonexistent_dir, nonexistent_file) self.assertEqual(abspath(bqplot_dir), guess_deploy_dir(bqplot_dir, None)) self.assertEqual(abspath(bqplot_dir), guess_deploy_dir(bqplot_ipynb, None)) self.assertEqual(abspath(bqplot_dir), guess_deploy_dir(bqplot_ipynb, bqplot_ipynb)) From d2c28ea14ae673c4603df95c5511239fe29bfdb3 Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Thu, 9 Mar 2023 17:32:24 -0500 Subject: [PATCH 12/16] optional absolute paths for validate_extra_files --- rsconnect/bundle.py | 6 +++--- tests/test_bundle.py | 41 +++++++++++++++++++++++++++++++++++++---- 2 files changed, 40 insertions(+), 7 deletions(-) diff --git a/rsconnect/bundle.py b/rsconnect/bundle.py index 28933598..39565c95 100644 --- a/rsconnect/bundle.py +++ b/rsconnect/bundle.py @@ -843,7 +843,6 @@ def make_api_manifest( def create_html_manifest( path: str, entrypoint: str, - app_mode: AppMode = AppModes.STATIC, extra_files: typing.List[str] = None, excludes: typing.List[str] = None, image: str = None, @@ -884,7 +883,7 @@ def create_html_manifest( raise RSConnectException("No valid entrypoint found.") entrypoint = abs_entrypoint(path, entrypoint) - extra_files = validate_extra_files(deploy_dir, extra_files) + extra_files = validate_extra_files(deploy_dir, extra_files, use_abspath=True) excludes = list(excludes) if excludes else [] excludes.extend(["manifest.json"]) excludes.extend(list_environment_dirs(deploy_dir)) @@ -1315,7 +1314,7 @@ def validate_file_is_notebook(file_name): raise RSConnectException("A Jupyter notebook (.ipynb) file is required here.") -def validate_extra_files(directory, extra_files): +def validate_extra_files(directory, extra_files, use_abspath=False): """ If the user specified a list of extra files, validate that they all exist and are beneath the given directory and, if so, return a list of them made relative to that @@ -1335,6 +1334,7 @@ def validate_extra_files(directory, extra_files): raise RSConnectException("%s must be under %s." % (extra_file, directory)) if not exists(join(directory, extra_file)): raise RSConnectException("Could not find file %s under %s" % (extra, directory)) + extra_file = abspath(join(directory, extra_file)) if use_abspath else extra_file result.append(extra_file) return result diff --git a/tests/test_bundle.py b/tests/test_bundle.py index 56ce740f..9216ed7b 100644 --- a/tests/test_bundle.py +++ b/tests/test_bundle.py @@ -835,7 +835,7 @@ def test_guess_deploy_dir(self): self.assertEqual(abspath(bqplot_dir), guess_deploy_dir(bqplot_dir, None)) self.assertEqual(abspath(bqplot_dir), guess_deploy_dir(bqplot_ipynb, None)) self.assertEqual(abspath(bqplot_dir), guess_deploy_dir(bqplot_ipynb, bqplot_ipynb)) - self.assertEqual(abspath(bqplot_dir), guess_deploy_dir(bqplot_dir, "bqplot.ipynb")) + self.assertEqual(abspath(bqplot_dir), guess_deploy_dir(bqplot_dir, bqplot_ipynb)) @pytest.mark.parametrize( @@ -1357,8 +1357,10 @@ def test_make_voila_bundle_2( single_file_nonindex_dir = os.path.join(cur_dir, "./testdata/html_tests/single_file_nonindex") multi_file_index_dir = os.path.join(cur_dir, "./testdata/html_tests/multi_file_index") multi_file_index_file = os.path.join(cur_dir, "./testdata/html_tests/multi_file_index/index.html") +multi_file_index_file2 = os.path.join(cur_dir, "./testdata/html_tests/multi_file_index/main.html") multi_file_nonindex_dir = os.path.join(cur_dir, "./testdata/html_tests/multi_file_nonindex") -multi_file_nonindex_file = os.path.join(cur_dir, "./testdata/html_tests/multi_file_nonindex/b.html") +multi_file_nonindex_fileb = os.path.join(cur_dir, "./testdata/html_tests/multi_file_nonindex/b.html") +multi_file_nonindex_filea = os.path.join(cur_dir, "./testdata/html_tests/multi_file_nonindex/a.html") def test_create_html_manifest(): @@ -1460,7 +1462,7 @@ def test_create_html_manifest(): } manifest = create_html_manifest( - multi_file_nonindex_file, + multi_file_nonindex_fileb, None, ) assert multi_file_nonindex_file_ans == json.loads(manifest.flattened_copy.json) @@ -1476,6 +1478,37 @@ def test_create_html_manifest(): manifest = create_html_manifest( multi_file_nonindex_dir, - multi_file_nonindex_file, + multi_file_nonindex_fileb, ) assert multi_file_nonindex_dir_and_file_ans == json.loads(manifest.flattened_copy.json) + + multi_file_nonindex_file_extras_ans = { + "version": 1, + "metadata": {"appmode": "static", "primary_html": "b.html", "entrypoint": "b.html"}, + "files": { + "a.html": {"checksum": "c14bd63e50295f94b761ffe9d41e3742"}, + "b.html": {"checksum": "c14bd63e50295f94b761ffe9d41e3742"}, + }, + } + manifest = create_html_manifest( + multi_file_nonindex_fileb, + None, + extra_files=[multi_file_nonindex_filea], + ) + assert multi_file_nonindex_file_extras_ans == json.loads(manifest.flattened_copy.json) + + multi_file_index_dir_extras_ans = { + "version": 1, + "metadata": {"appmode": "static", "primary_html": "index.html", "entrypoint": "index.html"}, + "files": { + "index.html": {"checksum": "c14bd63e50295f94b761ffe9d41e3742"}, + "main.html": {"checksum": "c14bd63e50295f94b761ffe9d41e3742"}, + }, + } + + manifest = create_html_manifest( + multi_file_index_dir, + None, + extra_files=[multi_file_index_file2], + ) + assert multi_file_index_dir_extras_ans == json.loads(manifest.flattened_copy.json) From 0d537960bcc36af7dd3de920a57ebfff533e51bf Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Thu, 9 Mar 2023 18:10:56 -0500 Subject: [PATCH 13/16] add use_abspath flag to create_file_list remove create_abspath_list --- rsconnect/bundle.py | 61 ++++++++------------------------------------- 1 file changed, 11 insertions(+), 50 deletions(-) diff --git a/rsconnect/bundle.py b/rsconnect/bundle.py index 39565c95..c07ba509 100644 --- a/rsconnect/bundle.py +++ b/rsconnect/bundle.py @@ -891,7 +891,7 @@ def create_html_manifest( manifest = Manifest(app_mode=AppModes.STATIC, entrypoint=entrypoint, primary_html=entrypoint, image=image) manifest.deploy_dir = deploy_dir - file_list = create_abspath_list(path, extra_files, excludes) + file_list = create_file_list(path, extra_files, excludes, use_abspath=True) for abs_path in file_list: manifest.add_file(abs_path) @@ -944,6 +944,7 @@ def create_file_list( path: str, extra_files: typing.List[str] = None, excludes: typing.List[str] = None, + use_abspath: bool = False, ) -> typing.List[str]: """ Builds a full list of files under the given path that should be included @@ -962,7 +963,8 @@ def create_file_list( file_set = set(extra_files) # type: typing.Set[str] if isfile(path): - file_set.add(path) + path_to_add = abspath(path) if use_abspath else path + file_set.add(path_to_add) return sorted(file_set) for cur_dir, sub_dirs, files in os.walk(path): @@ -971,57 +973,16 @@ def create_file_list( if any(parent in exclude_paths for parent in Path(cur_dir).parents): continue for file in files: - abs_path = os.path.join(cur_dir, file) - rel_path = relpath(abs_path, path) + cur_path = os.path.join(cur_dir, file) + rel_path = relpath(cur_path, path) - if Path(abs_path) in exclude_paths: + if Path(cur_path) in exclude_paths: continue if keep_manifest_specified_file(rel_path, exclude_paths | directories_to_ignore) and ( - rel_path in extra_files or not glob_set.matches(abs_path) + rel_path in extra_files or not glob_set.matches(cur_path) ): - file_set.add(rel_path) - return sorted(file_set) - - -def create_abspath_list( - path: str, - extra_files: typing.List[str] = None, - excludes: typing.List[str] = None, -) -> typing.List[str]: - """ - Builds a absolute path list of files under the given path that should be included - in a manifest or bundle. - - :param path: a file, or a directory to walk for files. - :param extra_files: a sequence of any extra files to include in the bundle. - :param excludes: a sequence of glob patterns that will exclude matched files. - :return: the list of relevant files, relative to the given directory. - """ - extra_files = extra_files or [] - excludes = excludes if excludes else [] - glob_set = create_glob_set(path, excludes) - exclude_paths = {Path(p) for p in excludes} - file_set = set(extra_files) # type: typing.Set[str] - - if isfile(path): - file_set.add(abspath(path)) - return sorted(file_set) - - for cur_dir, sub_dirs, files in os.walk(path): - if Path(cur_dir) in exclude_paths: - continue - if any(parent in exclude_paths for parent in Path(cur_dir).parents): - continue - for file in files: - abs_path = os.path.join(cur_dir, file) - rel_path = relpath(abs_path, path) - - if Path(abs_path) in exclude_paths: - continue - if keep_manifest_specified_file(rel_path, exclude_paths | directories_to_ignore) and ( - rel_path in extra_files or not glob_set.matches(abs_path) - ): - file_set.add(abspath(abs_path)) + path_to_add = abspath(cur_path) if use_abspath else rel_path + file_set.add(path_to_add) return sorted(file_set) @@ -1733,7 +1694,7 @@ def create_voila_manifest( manifest.add_to_buffer(join(deploy_dir, environment.filename), environment.contents) - file_list = create_abspath_list(path, extra_files, excludes) + file_list = create_file_list(path, extra_files, excludes, use_abspath=True) for abs_path in file_list: manifest.add_file(abs_path) return manifest From 4a627f98abe439e911956027ff2e95143e60d177 Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Fri, 10 Mar 2023 12:04:53 -0500 Subject: [PATCH 14/16] add unit tests for make_html_bundle --- tests/test_bundle.py | 203 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 203 insertions(+) diff --git a/tests/test_bundle.py b/tests/test_bundle.py index 9216ed7b..3c65f492 100644 --- a/tests/test_bundle.py +++ b/tests/test_bundle.py @@ -18,6 +18,7 @@ get_python_env_info, inspect_environment, list_files, + make_html_bundle, make_manifest_bundle, make_notebook_html_bundle, make_notebook_source_bundle, @@ -1512,3 +1513,205 @@ def test_create_html_manifest(): extra_files=[multi_file_index_file2], ) assert multi_file_index_dir_extras_ans == json.loads(manifest.flattened_copy.json) + + +def test_make_html_bundle(): + single_file_index_file_ans = { + "version": 1, + "metadata": {"appmode": "static", "primary_html": "index.html", "entrypoint": "index.html"}, + "files": {"index.html": {"checksum": "c14bd63e50295f94b761ffe9d41e3742"}}, + } + with make_html_bundle( + single_file_index_file, + None, + None, + None, + ) as bundle, tarfile.open(mode="r:gz", fileobj=bundle) as tar: + names = sorted(tar.getnames()) + assert names == [ + "index.html", + "manifest.json", + ] + assert single_file_index_file_ans == json.loads(tar.extractfile("manifest.json").read().decode("utf-8")) + + single_file_index_dir_ans = { + "version": 1, + "metadata": {"appmode": "static", "primary_html": "index.html", "entrypoint": "index.html"}, + "files": { + "index.html": {"checksum": "c14bd63e50295f94b761ffe9d41e3742"}, + "test1.txt": {"checksum": "3e7705498e8be60520841409ebc69bc1"}, + "test_folder1/testfoldertext1.txt": {"checksum": "0a576fd324b6985bac6aa934131d2f5c"}, + }, + } + with make_html_bundle( + single_file_index_dir, + None, + None, + None, + ) as bundle, tarfile.open(mode="r:gz", fileobj=bundle) as tar: + names = sorted(tar.getnames()) + assert names == [ + "index.html", + "manifest.json", + "test1.txt", + "test_folder1/testfoldertext1.txt", + ] + assert single_file_index_dir_ans == json.loads(tar.extractfile("manifest.json").read().decode("utf-8")) + + with make_html_bundle( + single_file_index_dir, + single_file_index_file, + None, + None, + ) as bundle, tarfile.open(mode="r:gz", fileobj=bundle) as tar: + names = sorted(tar.getnames()) + assert names == [ + "index.html", + "manifest.json", + "test1.txt", + "test_folder1/testfoldertext1.txt", + ] + assert single_file_index_dir_ans == json.loads(tar.extractfile("manifest.json").read().decode("utf-8")) + + multi_file_index_file_ans = { + "version": 1, + "metadata": {"appmode": "static", "primary_html": "index.html", "entrypoint": "index.html"}, + "files": {"index.html": {"checksum": "c14bd63e50295f94b761ffe9d41e3742"}}, + } + with make_html_bundle( + multi_file_index_file, + None, + None, + None, + ) as bundle, tarfile.open(mode="r:gz", fileobj=bundle) as tar: + names = sorted(tar.getnames()) + assert names == [ + "index.html", + "manifest.json", + ] + assert multi_file_index_file_ans == json.loads(tar.extractfile("manifest.json").read().decode("utf-8")) + + multi_file_index_dir_ans = { + "version": 1, + "metadata": {"appmode": "static", "primary_html": "index.html", "entrypoint": "index.html"}, + "files": { + "index.html": {"checksum": "c14bd63e50295f94b761ffe9d41e3742"}, + "main.html": {"checksum": "c14bd63e50295f94b761ffe9d41e3742"}, + }, + } + with make_html_bundle( + multi_file_index_dir, + None, + None, + None, + ) as bundle, tarfile.open(mode="r:gz", fileobj=bundle) as tar: + names = sorted(tar.getnames()) + assert names == [ + "index.html", + "main.html", + "manifest.json", + ] + assert multi_file_index_dir_ans == json.loads(tar.extractfile("manifest.json").read().decode("utf-8")) + + with make_html_bundle( + multi_file_index_dir, + multi_file_index_file, + None, + None, + ) as bundle, tarfile.open(mode="r:gz", fileobj=bundle) as tar: + names = sorted(tar.getnames()) + assert names == [ + "index.html", + "main.html", + "manifest.json", + ] + assert multi_file_index_dir_ans == json.loads(tar.extractfile("manifest.json").read().decode("utf-8")) + + multi_file_nonindex_file_ans = { + "version": 1, + "metadata": {"appmode": "static", "primary_html": "b.html", "entrypoint": "b.html"}, + "files": {"b.html": {"checksum": "c14bd63e50295f94b761ffe9d41e3742"}}, + } + with make_html_bundle( + multi_file_nonindex_fileb, + None, + None, + None, + ) as bundle, tarfile.open(mode="r:gz", fileobj=bundle) as tar: + names = sorted(tar.getnames()) + assert names == [ + "b.html", + "manifest.json", + ] + assert multi_file_nonindex_file_ans == json.loads(tar.extractfile("manifest.json").read().decode("utf-8")) + + multi_file_nonindex_dir_and_file_ans = { + "version": 1, + "metadata": {"appmode": "static", "primary_html": "b.html", "entrypoint": "b.html"}, + "files": { + "a.html": {"checksum": "c14bd63e50295f94b761ffe9d41e3742"}, + "b.html": {"checksum": "c14bd63e50295f94b761ffe9d41e3742"}, + }, + } + with make_html_bundle( + multi_file_nonindex_dir, + multi_file_nonindex_fileb, + None, + None, + ) as bundle, tarfile.open(mode="r:gz", fileobj=bundle) as tar: + names = sorted(tar.getnames()) + assert names == [ + "a.html", + "b.html", + "manifest.json", + ] + assert multi_file_nonindex_dir_and_file_ans == json.loads( + tar.extractfile("manifest.json").read().decode("utf-8") + ) + + multi_file_nonindex_file_extras_ans = { + "version": 1, + "metadata": {"appmode": "static", "primary_html": "b.html", "entrypoint": "b.html"}, + "files": { + "a.html": {"checksum": "c14bd63e50295f94b761ffe9d41e3742"}, + "b.html": {"checksum": "c14bd63e50295f94b761ffe9d41e3742"}, + }, + } + with make_html_bundle( + multi_file_nonindex_fileb, + None, + [multi_file_nonindex_filea], + None, + ) as bundle, tarfile.open(mode="r:gz", fileobj=bundle) as tar: + names = sorted(tar.getnames()) + assert names == [ + "a.html", + "b.html", + "manifest.json", + ] + assert multi_file_nonindex_file_extras_ans == json.loads( + tar.extractfile("manifest.json").read().decode("utf-8") + ) + + multi_file_index_dir_extras_ans = { + "version": 1, + "metadata": {"appmode": "static", "primary_html": "index.html", "entrypoint": "index.html"}, + "files": { + "index.html": {"checksum": "c14bd63e50295f94b761ffe9d41e3742"}, + "main.html": {"checksum": "c14bd63e50295f94b761ffe9d41e3742"}, + }, + } + + with make_html_bundle( + multi_file_index_dir, + None, + [multi_file_index_file2], + None, + ) as bundle, tarfile.open(mode="r:gz", fileobj=bundle) as tar: + names = sorted(tar.getnames()) + assert names == [ + "index.html", + "main.html", + "manifest.json", + ] + assert multi_file_index_dir_extras_ans == json.loads(tar.extractfile("manifest.json").read().decode("utf-8")) From 5735481e8fd4aeab773f273708dbda0bafbc08cb Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Fri, 10 Mar 2023 12:25:22 -0500 Subject: [PATCH 15/16] Update changelog --- CHANGELOG.md | 61 +++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 60 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a640b00a..819176d4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,8 +7,67 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased ### Added - - The `CONNECT_REQUEST_TIMEOUT` environment variable, which configures the request timeout for all blocking HTTP and HTTPS operations. This value translates into seconds (e.g., `CONNECT_REQUEST_TIMEOUT=60` is equivalent to 60 seconds.) By default, this value is 300. +- Added `deploy voila` command to deploy Jupyter Voila notebooks. + +### Changed +- `deploy html` was refactored. Its behavior is described below. + +#### Deploying HTML +Specifying a directory in the path will result in that entire directory*, subdirectories, and sub contents included in the deploy bundle. The entire directory is included whether or not an entrypoint was supplied + + +e.g. +using the following directory, +``` +├─ my_project/ +│ ├─ index.html +│ ├─ second.html +``` +and the following command: +``` +rsconnect deploy html -n local my_project +``` +or this command: +``` +rsconnect deploy html -n local my_project -e my_project/index.html +``` +we will have a bundle which includes both `index.html` and `second.html` + +- specifying a file in the path will result in that file* - not the entire directory - included in the deploy bundle + +e.g. +using the following directory, +``` +├─ my_project/ +│ ├─ index.html +│ ├─ second.html +``` +and the following command: +``` +rsconnect deploy html -n local my_project/second.html +``` +we will have a bundle which includes `second.html` + +- a note regarding entrypiont + - providing an entrypoint is optional if there's an `index.html` inside the project directory, or if there's a *single* html file in the project directory. + - if there are multiple html files in the project directory and it contains no `index.html`, we will get an exception when deploying that directory unless an entrypoint is specified. + +- if we want to specify an entrypint, and we are executing the deploy command outside a project folder, we must specify the full path of the entrypoint: + +``` +rsconnect deploy html -n local my_project -e my_project/second.html +``` + +- if we want to specify an entrypint, and we are executing the deploy command inside the project folder, we can abbreviate the entrypoint, like so: +``` +cd my_project +rsconnect deploy html -n local ./ -e second.html +``` + + +*Plus the manifest & other necessary files needed for the bundle to work on Connect. + ## [1.14.1] - 2023-02-09 From bad3188838e5572d443d9c9fb5c6b53d9e8402c3 Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Fri, 10 Mar 2023 13:57:38 -0500 Subject: [PATCH 16/16] correct description --- rsconnect/bundle.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rsconnect/bundle.py b/rsconnect/bundle.py index c07ba509..9824ce95 100644 --- a/rsconnect/bundle.py +++ b/rsconnect/bundle.py @@ -906,7 +906,7 @@ def make_html_bundle( image: str = None, ) -> typing.IO[bytes]: """ - Create an voila bundle, given a path and/or entrypoint. + Create an html bundle, given a path and/or entrypoint. The bundle contains a manifest.json file created for the given notebook entrypoint file. If the related environment file (requirements.txt) doesn't