From 0e76188163b95b03f77465cb7a08cc5c57498000 Mon Sep 17 00:00:00 2001 From: Matthew Lynch Date: Wed, 22 Jun 2022 15:56:45 -0500 Subject: [PATCH 01/57] initial work to support shinyapps.io hosting --- rsconnect/actions.py | 37 +++-- rsconnect/actions_content.py | 12 +- rsconnect/api.py | 283 ++++++++++++++++++++++++++++++----- rsconnect/http_support.py | 29 +++- rsconnect/main.py | 203 ++++++++++++++++--------- rsconnect/metadata.py | 75 +++++++--- tests/test_api.py | 6 +- tests/test_main.py | 46 +++++- tests/test_metadata.py | 63 +++++--- 9 files changed, 584 insertions(+), 170 deletions(-) diff --git a/rsconnect/actions.py b/rsconnect/actions.py index 4ddfe3ce..79a1a3d2 100644 --- a/rsconnect/actions.py +++ b/rsconnect/actions.py @@ -304,7 +304,7 @@ def check_server_capabilities(connect_server, capability_functions, details_sour raise api.RSConnectException(message) -def _make_deployment_name(connect_server, title, force_unique) -> str: +def _make_deployment_name(remote_server: api.RemoteServer, title: str, force_unique: bool) -> str: """ Produce a name for a deployment based on its title. It is assumed that the title is already defaulted and validated as appropriate (meaning the title @@ -315,7 +315,7 @@ def _make_deployment_name(connect_server, title, force_unique) -> str: that we collapse repeating underscores and, if the name is too short, it is padded to the left with underscores. - :param connect_server: the information needed to interact with the Connect server. + :param remote_server: the information needed to interact with the Connect server. :param title: the title to start with. :param force_unique: a flag noting whether the generated name must be forced to be unique. @@ -327,7 +327,12 @@ def _make_deployment_name(connect_server, title, force_unique) -> str: # Now, make sure it's unique, if needed. if force_unique: - name = api.find_unique_name(connect_server, name) + if isinstance(remote_server, api.RSConnectServer): + name = api.find_unique_name(remote_server, name) + else: + # TODO (mslynch): is this necessary to check? + pass + # raise NotImplementedError("TODO (mslynch): find unique name from lucid-server") return name @@ -1384,7 +1389,7 @@ def _generate_gather_basic_deployment_info_for_python(app_mode: AppMode) -> typi """ def gatherer( - connect_server: api.RSConnectServer, + remote_server: api.RemoteServer, app_store: AppStore, directory: str, entry_point: str, @@ -1393,7 +1398,7 @@ def gatherer( title: str, ) -> typing.Tuple[str, int, str, str, bool, AppMode]: return _gather_basic_deployment_info_for_framework( - connect_server, + remote_server, app_store, directory, entry_point, @@ -1414,7 +1419,7 @@ def gatherer( def _gather_basic_deployment_info_for_framework( - connect_server: api.RSConnectServer, + remote_server: api.RemoteServer, app_store: AppStore, directory: str, entry_point: str, @@ -1426,7 +1431,7 @@ def _gather_basic_deployment_info_for_framework( """ Helps to gather the necessary info for performing a deployment. - :param connect_server: the Connect server information. + :param remote_server: the server information. :param app_store: the store for the specified directory. :param directory: the primary file being deployed. :param entry_point: the entry point for the API in ': format. if @@ -1451,13 +1456,17 @@ def _gather_basic_deployment_info_for_framework( if app_id is None: # Possible redeployment - check for saved metadata. # Use the saved app information unless overridden by the user. - app_id, existing_app_mode = app_store.resolve(connect_server.url, app_id, app_mode) + app_id, existing_app_mode = app_store.resolve(remote_server.url, app_id, app_mode) logger.debug("Using app mode from app %s: %s" % (app_id, app_mode)) elif app_id is not None: # Don't read app metadata if app-id is specified. Instead, we need # to get this from Connect. - app = api.get_app_info(connect_server, app_id) - existing_app_mode = AppModes.get_by_ordinal(app.get("app_mode", 0), True) + if isinstance(remote_server, api.RSConnectServer): + app = api.get_app_info(remote_server, app_id) + existing_app_mode = AppModes.get_by_ordinal(app.get("app_mode", 0), True) + else: + # TODO this + raise NotImplementedError("TODO (mslynch): get app mode from lucid-server") if existing_app_mode and app_mode != existing_app_mode: msg = ( "Deploying with mode '%s',\n" @@ -1475,7 +1484,7 @@ def _gather_basic_deployment_info_for_framework( return ( entry_point, app_id, - _make_deployment_name(connect_server, title, app_id is None), + _make_deployment_name(remote_server, title, app_id is None), title, default_title, app_mode, @@ -1634,7 +1643,7 @@ def create_quarto_deployment_bundle( def deploy_bundle( - connect_server: api.RSConnectServer, + remote_server: api.RemoteServer, app_id: int, name: str, title: str, @@ -1645,7 +1654,7 @@ def deploy_bundle( """ Deploys the specified bundle. - :param connect_server: the Connect server information. + :param remote_server: the server information. :param app_id: the ID of the app to deploy, if this is a redeploy. :param name: the name for the deploy. :param title: the title for the deploy. @@ -1655,7 +1664,7 @@ def deploy_bundle( :return: application information about the deploy. This includes the ID of the task that may be queried for deployment progress. """ - return api.do_bundle_deploy(connect_server, app_id, name, title, title_is_default, bundle, env_vars) + return api.do_bundle_deploy(remote_server, app_id, name, title, title_is_default, bundle, env_vars) def spool_deployment_log(connect_server, app, log_callback): diff --git a/rsconnect/actions_content.py b/rsconnect/actions_content.py index d1b797a3..2c42158b 100644 --- a/rsconnect/actions_content.py +++ b/rsconnect/actions_content.py @@ -10,7 +10,7 @@ import semver -from .api import RSConnect, RSConnectException, emit_task_log +from .api import RSConnectClient, RSConnectException, emit_task_log from .log import logger from .models import BuildStatus, ContentGuidWithBundle from .metadata import ContentBuildStore @@ -36,7 +36,7 @@ def build_add_content(connect_server, content_guids_with_bundle): + "please wait for it to finish before adding new content." ) - with RSConnect(connect_server, timeout=120) as client: + with RSConnectClient(connect_server, timeout=120) as client: if len(content_guids_with_bundle) == 1: all_content = [client.content_get(content_guids_with_bundle[0].guid)] else: @@ -227,7 +227,7 @@ def _monitor_build(connect_server, content_items): def _build_content_item(connect_server, content, poll_wait): init_content_build_store(connect_server) - with RSConnect(connect_server) as client: + with RSConnectClient(connect_server) as client: # Pending futures will still try to execute when ThreadPoolExecutor.shutdown() is called # so just exit immediately if the current build has been aborted. # ThreadPoolExecutor.shutdown(cancel_futures=) isnt available until py3.9 @@ -291,7 +291,7 @@ def download_bundle(connect_server, guid_with_bundle): """ :param guid_with_bundle: models.ContentGuidWithBundle """ - with RSConnect(connect_server, timeout=120) as client: + with RSConnectClient(connect_server, timeout=120) as client: # bundle_id not provided so grab the latest if not guid_with_bundle.bundle_id: content = client.get_content(guid_with_bundle.guid) @@ -310,7 +310,7 @@ def get_content(connect_server, guid): :param guid: a single guid as a string or list of guids. :return: a list of content items. """ - with RSConnect(connect_server, timeout=120) as client: + with RSConnectClient(connect_server, timeout=120) as client: if isinstance(guid, str): result = [client.get_content(guid)] else: @@ -321,7 +321,7 @@ def get_content(connect_server, guid): def search_content( connect_server, published, unpublished, content_type, r_version, py_version, title_contains, order_by ): - with RSConnect(connect_server, timeout=120) as client: + with RSConnectClient(connect_server, timeout=120) as client: result = client.search_content() result = _apply_content_filters( result, published, unpublished, content_type, r_version, py_version, title_contains diff --git a/rsconnect/api.py b/rsconnect/api.py index 73d2532a..06046964 100644 --- a/rsconnect/api.py +++ b/rsconnect/api.py @@ -1,9 +1,20 @@ """ RStudio Connect API client and utility functions """ - +import abc +import base64 +import calendar +import datetime +import hashlib +import hmac import time +import typing +import webbrowser from _ssl import SSLError +from urllib import parse +from urllib.parse import urlparse + +import click from .http_support import HTTPResponse, HTTPServer, append_to_path, CookieJar from .log import logger @@ -17,19 +28,18 @@ def __init__(self, message, cause=None): self.cause = cause -class RSConnectServer(object): - """ - A simple class to encapsulate the information needed to interact with an - instance of the Connect server. - """ - - def __init__(self, url, api_key, insecure=False, ca_data=None): - self.url = url - self.api_key = api_key - self.insecure = insecure - self.ca_data = ca_data - # This is specifically not None. - self.cookie_jar = CookieJar() +class AbstractRemoteServer: + # @property + # @abc.abstractmethod + # def url(self) -> str: + # pass + # + # @property + # @abc.abstractmethod + # def remote_name(self) -> str: + # pass + url: str + remote_name: str def handle_bad_response(self, response): if isinstance(response, HTTPResponse): @@ -41,21 +51,56 @@ def handle_bad_response(self, response): # search page so trap that since we know we're expecting JSON from Connect. This # also catches all error conditions which we will report as "not running Connect". else: - if response.json_data and "error" in response.json_data: - error = "The Connect server reported an error: %s" % response.json_data["error"] + if response.json_data and "error" in response.json_data and response.json_data["error"] is not None: + error = "%s reported an error: %s" % (self.remote_name, response.json_data["error"]) raise RSConnectException(error) if response.status < 200 or response.status > 299: raise RSConnectException( - "Received an unexpected response from RStudio Connect: %s %s" - % (response.status, response.reason) + "Received an unexpected response from %s: %s %s" + % (self.remote_name, response.status, response.reason) ) -class RSConnect(HTTPServer): - def __init__(self, server, cookies=None, timeout=30): +class ShinyappsServer(AbstractRemoteServer): + """ + A simple class to encapsulate the information needed to interact with an + instance of the shinyapps.io server. + """ + + remote_name = "shinyapps.io" + + def __init__(self, url: str, account_name: str, token: str, secret: str): + self.url = url + self.account_name = account_name + self.token = token + self.secret = secret + + +class RSConnectServer(AbstractRemoteServer): + """ + A simple class to encapsulate the information needed to interact with an + instance of the Connect server. + """ + + remote_name = "RStudio Connect" + + def __init__(self, url, api_key, insecure=False, ca_data=None): + self.url = url + self.api_key = api_key + self.insecure = insecure + self.ca_data = ca_data + # This is specifically not None. + self.cookie_jar = CookieJar() + + +RemoteServer = typing.Union[ShinyappsServer, RSConnectServer] + + +class RSConnectClient(HTTPServer): + def __init__(self, server: RSConnectServer, cookies=None, timeout=30): if cookies is None: cookies = server.cookie_jar - super(RSConnect, self).__init__( + super().__init__( append_to_path(server.url, "__api__"), server.insecure, server.ca_data, @@ -268,6 +313,149 @@ def output_task_log(task_status, last_status, log_callback): return new_last_status +class S3Client(HTTPServer): + def upload(self, path, presigned_checksum, bundle_size, contents): + headers = { + "content-type": "application/x-tar", + "content-length": str(bundle_size), + "content-md5": presigned_checksum, + } + return self.put(path, headers=headers, body=contents, decode_response=False) + + +class ShinyappsClient(HTTPServer): + def __init__(self, shinyapps_server: ShinyappsServer, timeout: int = 30): + self._token = shinyapps_server.token + self._key = base64.b64decode(shinyapps_server.secret) + self._server = shinyapps_server + super().__init__(shinyapps_server.url, timeout=timeout) + + def _get_canonical_request(self, method, path, timestamp, content_hash): + return "\n".join([method, path, timestamp, content_hash]) + + def _get_canonical_request_signature(self, request): + result = hmac.new(self._key, request.encode(), hashlib.sha256).hexdigest() + return base64.b64encode(result.encode()).decode() + + def get_extra_headers(self, url, method, body): + canonical_request_method = method.upper() + canonical_request_path = parse.urlparse(url).path + + # TODO (mslynch): there has got to be a better way than this + timetuple = datetime.datetime.utcnow().utctimetuple() + canonical_request_date = "%s, %02d %s %04d %02d:%02d:%02d %s" % ( + ["Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"][timetuple[6]], + timetuple[2], + ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"][timetuple[1] - 1], + timetuple[0], + timetuple[3], + timetuple[4], + timetuple[5], + "GMT", + ) + + # get request checksum + md5 = hashlib.md5() + body = body or b"" + body_bytes = body if isinstance(body, bytes) else body.encode() + md5.update(body_bytes) + canonical_request_checksum = md5.hexdigest() + + canonical_request = self._get_canonical_request( + canonical_request_method, canonical_request_path, canonical_request_date, canonical_request_checksum + ) + + signature = self._get_canonical_request_signature(canonical_request) + + return { + "X-Auth-Token": "{0}".format(self._token), + "X-Auth-Signature": "{0}; version=1".format(signature), + "Date": canonical_request_date, + "X-Content-Checksum": canonical_request_checksum, + } + + def create_application(self, account_id, application_name): + application_data = { + "account": account_id, + "name": application_name, + "template": "shiny", + } + return self.post("/v1/applications/", body=application_data) + + def get_accounts(self): + return self.get("/v1/accounts/") + + # def get_current_user(self): + # return self.get('v1/users/me') + + def create_bundle(self, application_id: int, content_type: str, content_length: int, checksum: str): + bundle_data = { + "application": application_id, + "content_type": content_type, + "content_length": content_length, + "checksum": checksum, + } + return self.post("/v1/bundles", body=bundle_data) + + def set_bundle_status(self, bundle_id, bundle_status): + return self.post(f"/v1/bundles/{bundle_id}/status", body={"status": bundle_status}) + + def deploy_application(self, bundle_id, app_id): + return self.post(f"/v1/applications/{app_id}/deploy", body={"bundle": bundle_id, "rebuild": False}) + + def get_task(self, task_id): + return self.get(f"/v1/tasks/{task_id}", query_params={"legacy": "true"}) + + def wait_until_task_is_successful(self, task_id, timeout=60): + counter = 1 + status = None + + while counter < timeout and status not in ["success", "failed", "error"]: + task = self.get_task(task_id) + self._server.handle_bad_response(task) + status = task.json_data["status"] + description = task.json_data["description"] + + click.secho(f"Waiting: {status} - {description}") + + if status == "success": + break + + time.sleep(2) + counter += 1 + click.secho(f"Task done: {description}") + + def prepare_deploy(self, app_id, app_name, app_title, title_is_default, bundle_size, bundle_hash, env_vars=None): + accounts = self.get_accounts() + self._server.handle_bad_response(accounts) + account = next( + filter(lambda account: account["name"] == self._server.account_name, accounts.json_data["accounts"]), None + ) + # TODO: also check this during `add` command + if account is None: + raise RSConnectException( + "No account found by name : %s for given user credential" % self._server.account_name + ) + + application = self.create_application(account["id"], app_name) + self._server.handle_bad_response(application) + + bundle = self.create_bundle(application.json_data["id"], "application/x-tar", bundle_size, bundle_hash) + + # TODO (mslynch) _server isn't meant to handle S3 errors + self._server.handle_bad_response(bundle) + + return {"app_id": application.json_data["id"], "app_url": application.json_data["url"], **bundle.json_data} + + def do_deploy(self, bundle_id, app_id): + bundle_status_response = self.set_bundle_status(bundle_id, "ready") + self._server.handle_bad_response(bundle_status_response) + + deploy_task = self.deploy_application(bundle_id, app_id) + self._server.handle_bad_response(deploy_task) + self.wait_until_task_is_successful(deploy_task.json_data["id"]) + + def verify_server(connect_server): """ Verify that the given server information represents a Connect instance that is @@ -278,7 +466,7 @@ def verify_server(connect_server): :return: the server settings from the Connect server. """ try: - with RSConnect(connect_server) as client: + with RSConnectClient(connect_server) as client: result = client.server_settings() connect_server.handle_bad_response(result) return result @@ -294,7 +482,7 @@ def verify_api_key(connect_server): :param connect_server: the Connect server information, including the API key to test. :return: the username of the user to whom the API key belongs. """ - with RSConnect(connect_server) as client: + with RSConnectClient(connect_server) as client: result = client.me() if isinstance(result, HTTPResponse): if result.json_data and "code" in result.json_data and result.json_data["code"] == 30: @@ -311,7 +499,7 @@ def get_python_info(connect_server): :param connect_server: the Connect server information. :return: the Python installation information from Connect. """ - with RSConnect(connect_server) as client: + with RSConnectClient(connect_server) as client: result = client.python_settings() connect_server.handle_bad_response(result) return result @@ -325,7 +513,7 @@ def get_app_info(connect_server, app_id): :param app_id: the ID (numeric or GUID) of the application to get info for. :return: the Python installation information from Connect. """ - with RSConnect(connect_server) as client: + with RSConnectClient(connect_server) as client: result = client.app_get(app_id) connect_server.handle_bad_response(result) return result @@ -340,17 +528,17 @@ def get_app_config(connect_server, app_id): :param app_id: the ID (numeric or GUID) of the application to get the info for. :return: the Python installation information from Connect. """ - with RSConnect(connect_server) as client: + with RSConnectClient(connect_server) as client: result = client.app_config(app_id) connect_server.handle_bad_response(result) return result -def do_bundle_deploy(connect_server, app_id, name, title, title_is_default, bundle, env_vars): +def do_bundle_deploy(remote_server: RemoteServer, app_id, name, title, title_is_default, bundle, env_vars): """ Deploys the specified bundle. - :param connect_server: the Connect server information. + :param remote_server: the server information. :param app_id: the ID of the app to deploy, if this is a redeploy. :param name: the name for the deploy. :param title: the title for the deploy. @@ -360,10 +548,37 @@ def do_bundle_deploy(connect_server, app_id, name, title, title_is_default, bund :return: application information about the deploy. This includes the ID of the task that may be queried for deployment progress. """ - with RSConnect(connect_server, timeout=120) as client: - result = client.deploy(app_id, name, title, title_is_default, bundle, env_vars) - connect_server.handle_bad_response(result) - return result + if isinstance(remote_server, RSConnectServer): + with RSConnectClient(remote_server, timeout=120) as client: + result = client.deploy(app_id, name, title, title_is_default, bundle, env_vars) + remote_server.handle_bad_response(result) + return result + else: + contents = bundle.read() + bundle_size = len(contents) + bundle_hash = hashlib.md5(contents).hexdigest() + + with ShinyappsClient(remote_server, timeout=120) as client: + prepare_deploy_result = client.prepare_deploy( + app_id, name, title, title_is_default, bundle_size, bundle_hash, env_vars + ) + + parsed_upload_url = urlparse(prepare_deploy_result["presigned_url"]) + with S3Client(f"{parsed_upload_url.scheme}://{parsed_upload_url.netloc}", timeout=120) as client: + upload_result = client.upload( + prepare_deploy_result["presigned_url"], + prepare_deploy_result["presigned_checksum"], + bundle_size, + contents, + ) + remote_server.handle_bad_response(upload_result) + + with ShinyappsClient(remote_server, timeout=120) as client: + deploy_result = client.do_deploy(prepare_deploy_result["id"], prepare_deploy_result["app_id"]) + + webbrowser.open_new(prepare_deploy_result["app_url"]) + + return {"app_url": prepare_deploy_result["app_url"], "app_id": prepare_deploy_result["id"], "app_guid": None} def emit_task_log( @@ -393,7 +608,7 @@ def emit_task_log( :return: the ultimate URL where the deployed app may be accessed and the sequence of log lines. The log lines value will be None if a log callback was provided. """ - with RSConnect(connect_server) as client: + with RSConnectClient(connect_server) as client: result = client.wait_for_task(task_id, log_callback, abort_func, timeout, poll_wait, raise_on_error) connect_server.handle_bad_response(result) app_config = client.app_config(app_id) @@ -428,7 +643,7 @@ def retrieve_matching_apps(connect_server, filters=None, limit=None, mapping_fun maximum = limit finished = False - with RSConnect(connect_server) as client: + with RSConnectClient(connect_server) as client: while not finished: response = client.app_search(search_filters) connect_server.handle_bad_response(response) diff --git a/rsconnect/http_support.py b/rsconnect/http_support.py index decab61d..52f6eaa4 100644 --- a/rsconnect/http_support.py +++ b/rsconnect/http_support.py @@ -225,14 +225,34 @@ def post(self, path, query_params=None, body=None): def patch(self, path, query_params=None, body=None): return self.request("PATCH", path, query_params, body) - def request(self, method, path, query_params=None, body=None, maximum_redirects=5, decode_response=True): + def put(self, path, query_params=None, body=None, headers=None, decode_response=True): + if headers is None: + headers = {} + return self.request( + "PUT", path, query_params=query_params, body=body, headers=headers, decode_response=decode_response + ) + + def request( + self, + method, + path, + query_params=None, + body=None, + maximum_redirects=5, + decode_response=True, + headers=None, + ): path = self._get_full_path(path) - extra_headers = None + extra_headers = headers or {} if isinstance(body, (dict, list)): body = json.dumps(body).encode("utf-8") extra_headers = {"Content-Type": "application/json; charset=utf-8"} + extra_headers = {**extra_headers, **self.get_extra_headers(path, method, body)} return self._do_request(method, path, query_params, body, maximum_redirects, extra_headers, decode_response) + def get_extra_headers(self, url, method, body): + return {} + def _do_request( self, method, path, query_params, body, maximum_redirects, extra_headers=None, decode_response=True ): @@ -287,13 +307,14 @@ def _do_request( logger.debug("--> Redirected to: %s" % next_url) + redirect_extra_headers = self.get_extra_headers(next_url, "GET", body) return self._do_request( - method, + "GET", next_url, query_params, body, maximum_redirects - 1, - extra_headers, + {**extra_headers, **redirect_extra_headers}, ) self._handle_set_cookie(response) diff --git a/rsconnect/main.py b/rsconnect/main.py index 645da90d..8cd7e1c4 100644 --- a/rsconnect/main.py +++ b/rsconnect/main.py @@ -226,7 +226,7 @@ def _test_server_and_api(server, api_key, insecure, ca_cert): # noinspection SpellCheckingInspection @cli.command( - short_help="Define a nickname for an RStudio Connect server.", + short_help="Define a nickname for an RStudio Connect or shinyapps.io server and credential.", help=( "Associate a simple nickname with the information needed to interact with an RStudio Connect server. " "Specifying an existing nickname will cause its stored information to be replaced by what is given " @@ -234,17 +234,23 @@ def _test_server_and_api(server, api_key, insecure, ca_cert): ), ) @click.option("--name", "-n", required=True, help="The nickname of the RStudio Connect server to deploy to.") +@click.option( + "--target", + "-t", + type=click.Choice(["connect", "shinyapps"]), + default="connect", + envvar="RSCONNECT_SERVER_TYPE", + help="The target platform of this credential (Connect or shinyapps.io).", +) @click.option( "--server", "-s", - required=True, envvar="CONNECT_SERVER", help="The URL for the RStudio Connect server to deploy to.", ) @click.option( "--api-key", "-k", - required=True, envvar="CONNECT_API_KEY", help="The API key to use to authenticate with RStudio Connect.", ) @@ -262,27 +268,79 @@ def _test_server_and_api(server, api_key, insecure, ca_cert): type=click.File(), help="The path to trusted TLS CA certificates.", ) +@click.option( + "--token", + "-T", + envvar="SHINYAPPS_TOKEN", + help="The shinyapps.io token.", +) +@click.option( + "--secret", + "-S", + envvar="SHINYAPPS_SECRET", + help="The shinyapps.io token secret.", +) @click.option("--verbose", "-v", is_flag=True, help="Print detailed messages.") -def add(name, server, api_key, insecure, cacert, verbose): +def add(name, target, server, api_key, insecure, cacert, token, secret, verbose): set_verbosity(verbose) old_server = server_store.get_by_name(name) - # Server must be pingable and the API key must work to be added. - real_server, _ = _test_server_and_api(server, api_key, insecure, cacert) - - server_store.set( - name, - real_server.url, - real_server.api_key, - real_server.insecure, - real_server.ca_data, - ) + # validate options + if target == "connect": + # # disallowed + # if token is not None: + # raise api.RSConnectException("Token may not be specified when using target type 'connect'.") + # if secret is not None: + # raise api.RSConnectException("Secret may not be specified when using target type 'connect'.") + + # required + if server is None: + raise api.RSConnectException("Server must be specified when using target type 'connect'.") + if api_key is None: + raise api.RSConnectException("API key must be specified when using target type 'connect'.") + else: + # # disallowed + # if api_key is not None: + # raise api.RSConnectException("API key may not be specified when using target type 'shinyapps'.") + # if insecure is not None: + # raise api.RSConnectException("Insecure may not be specified when using target type 'shinyapps'.") + # if cacert is not None: + # raise api.RSConnectException("CA certificate may not be specified when using target type 'shinyapps'.") + + # required + if token is None: + raise api.RSConnectException("Server must be specified when using target type 'shinyapps'.") + if secret is None: + raise api.RSConnectException("API key must be specified when using target type 'shinyapps'.") + + if target == "connect": + # Server must be pingable and the API key must work to be added. + real_server, _ = _test_server_and_api(server, api_key, insecure, cacert) + + server_store.set( + name, + target, + real_server.url, + real_server.api_key, + real_server.insecure, + real_server.ca_data, + ) - if old_server: - click.echo('Updated server "%s" with URL %s' % (name, real_server.url)) + if old_server: + click.echo('Updated Connect server "%s" with URL %s' % (name, real_server.url)) + else: + click.echo('Added Connect server "%s" with URL %s' % (name, real_server.url)) else: - click.echo('Added server "%s" with URL %s' % (name, real_server.url)) + shinyapps_server = api.ShinyappsServer(server or "https://api.shinyapps.io", name, token, secret) + # TODO (mslynch): check credential and account here + server_store.set( + name, target, shinyapps_server.url, token=shinyapps_server.token, secret=shinyapps_server.secret + ) + if old_server: + click.echo('Added shinyapps.io credential "%s".' % name) + else: + click.echo('Updated shinyapps.io credential "%s".' % name) @cli.command( @@ -459,7 +517,7 @@ def deploy(): pass -def _validate_deploy_to_args(name, url, api_key, insecure, ca_cert, api_key_is_required=True): +def _validate_deploy_to_args(name, url, api_key, insecure, ca_cert, api_key_is_required=True) -> api.RemoteServer: """ Validate that the user gave us enough information to talk to a Connect server. @@ -470,38 +528,44 @@ def _validate_deploy_to_args(name, url, api_key, insecure, ca_cert, api_key_is_r :param ca_cert: the name of a CA certs file containing certificates to use. :param api_key_is_required: a flag that notes whether the API key is required or may be omitted. - :return: a ConnectServer object that carries all the right info. + :return: a ConnectServer or ShinyappsServer object that carries all the right info. """ ca_data = ca_cert and text_type(ca_cert.read()) if name and url: raise api.RSConnectException("You must specify only one of -n/--name or -s/--server, not both.") - real_server, api_key, insecure, ca_data, from_store = server_store.resolve(name, url, api_key, insecure, ca_data) + server_data = server_store.resolve(name, url, api_key, insecure, ca_data) # This can happen if the user specifies neither --name or --server and there's not # a single default to go with. - if not real_server: + if not server_data.url: raise api.RSConnectException("You must specify one of -n/--name or -s/--server.") - connect_server = api.RSConnectServer(real_server, None, insecure, ca_data) + if server_data.target == "connect": - # If our info came from the command line, make sure the URL really works. - if not from_store: - connect_server, _ = test_server(connect_server) + connect_server = api.RSConnectServer(server_data.url, None, server_data.insecure, server_data.ca_data) - connect_server.api_key = api_key + # If our info came from the command line, make sure the URL really works. + if not server_data.from_store: + connect_server, _ = test_server(connect_server) - if not connect_server.api_key: - if api_key_is_required: - raise api.RSConnectException('An API key must be specified for "%s".' % connect_server.url) - return connect_server + connect_server.api_key = server_data.api_key - # If our info came from the command line, make sure the key really works. - if not from_store: - _ = test_api_key(connect_server) + if not connect_server.api_key: + if api_key_is_required: + raise api.RSConnectException('An API key must be specified for "%s".' % connect_server.url) + return connect_server - return connect_server + # If our info came from the command line, make sure the key really works. + if not server_data.from_store: + _ = test_api_key(connect_server) + + return connect_server + else: + # TODO (mslynch): validate shinyapps data + # TODO (mslynch): replace nickname with account name + return api.ShinyappsServer(server_data.url, server_data.name, server_data.token, server_data.secret) def _warn_on_ignored_manifest(directory): @@ -578,7 +642,7 @@ def _warn_on_ignored_requirements(directory, requirements_file_name): def _deploy_bundle( - connect_server, + remote_server: api.RemoteServer, app_store, primary_path, app_id, @@ -589,10 +653,10 @@ def _deploy_bundle( bundle, env_vars, ): - """ + """œ Does the work of uploading a prepared bundle. - :param connect_server: the Connect server information. + :param remote_server: the Connect server information. :param app_store: the store where data is saved about deployments. :param primary_path: the base path (file or directory) that's being deployed. :param app_id: the ID of the app. @@ -604,12 +668,12 @@ def _deploy_bundle( :param env_vars: list of NAME=VALUE pairs to be set as the app environment """ with cli_feedback("Uploading bundle"): - app = deploy_bundle(connect_server, app_id, name, title, title_is_default, bundle, env_vars) + app = deploy_bundle(remote_server, app_id, name, title, title_is_default, bundle, env_vars) with cli_feedback("Saving deployment data"): # Note we are NOT saving image into the deployment record for now. app_store.set( - connect_server.url, + remote_server.url, abspath(primary_path), app["app_url"], app["app_id"], @@ -618,26 +682,27 @@ def _deploy_bundle( app_mode, ) - with cli_feedback(""): - click.secho("\nDeployment log:") - app_url, _, _ = spool_deployment_log(connect_server, app, click.echo) - click.secho("Deployment completed successfully.") - click.secho(" Dashboard content URL: ", nl=False) - click.secho(app_url, fg="green") - click.secho(" Direct content URL: ", nl=False) - click.secho(app["app_url"], fg="green") - - # save the config URL, replacing the old app URL we got during deployment - # (which is the Open Solo URL). - app_store.set( - connect_server.url, - abspath(primary_path), - app_url, - app["app_id"], - app["app_guid"], - app["title"], - app_mode, - ) + if isinstance(remote_server, api.RSConnectServer): + with cli_feedback(""): + click.secho("\nDeployment log:") + app_url, _, _ = spool_deployment_log(remote_server, app, click.echo) + click.secho("Deployment completed successfully.") + click.secho(" Dashboard content URL: ", nl=False) + click.secho(app_url, fg="green") + click.secho(" Direct content URL: ", nl=False) + click.secho(app["app_url"], fg="green") + + # save the config URL, replacing the old app URL we got during deployment + # (which is the Open Solo URL). + app_store.set( + remote_server.url, + abspath(primary_path), + app_url, + app["app_id"], + app["app_guid"], + app["title"], + app_mode, + ) # noinspection SpellCheckingInspection,DuplicatedCode @@ -1262,15 +1327,15 @@ def _deploy_by_framework( set_verbosity(verbose) with cli_feedback("Checking arguments"): - connect_server = _validate_deploy_to_args(name, server, api_key, insecure, cacert) + remote_server = _validate_deploy_to_args(name, server, api_key, insecure, cacert) module_file = fake_module_file_from_directory(directory) extra_files = validate_extra_files(directory, extra_files) app_store = AppStore(module_file) entrypoint, app_id, deployment_name, title, default_title, app_mode = gatherer( - connect_server, app_store, directory, entrypoint, new, app_id, title + remote_server, app_store, directory, entrypoint, new, app_id, title ) - click.secho(' Deploying %s to server "%s"' % (directory, connect_server.url)) + click.secho(' Deploying %s to server "%s"' % (directory, remote_server.url)) _warn_on_ignored_manifest(directory) _warn_if_no_requirements_file(directory) @@ -1279,11 +1344,13 @@ def _deploy_by_framework( with cli_feedback("Inspecting Python environment"): _, environment = get_python_env_info(module_file, python, conda, force_generate) - with cli_feedback("Checking server capabilities"): - checks = [are_apis_supported_on_server] - if environment.package_manager == "conda": - checks.append(is_conda_supported_on_server) - check_server_capabilities(connect_server, checks) + # TODO (mslynch): check shinyapps capabilities? hard-code? skip? + if isinstance(remote_server, api.RSConnectServer): + with cli_feedback("Checking server capabilities"): + checks = [are_apis_supported_on_server] + if environment.package_manager == "conda": + checks.append(is_conda_supported_on_server) + check_server_capabilities(remote_server, checks) _warn_on_ignored_conda_env(environment) @@ -1296,7 +1363,7 @@ def _deploy_by_framework( ) _deploy_bundle( - connect_server, + remote_server, app_store, directory, app_id, diff --git a/rsconnect/metadata.py b/rsconnect/metadata.py index 72ba7a22..df16de4b 100644 --- a/rsconnect/metadata.py +++ b/rsconnect/metadata.py @@ -7,6 +7,7 @@ import os import glob import sys +import typing from datetime import datetime, timezone from os.path import abspath, basename, dirname, exists, join from urllib.parse import urlparse @@ -137,7 +138,7 @@ def _get_sorted_values(self, sort_by): """ Return all the values in the store sorted by the given lambda expression. - :param sort_by: a lambda expression to use to sort the values.. + :param sort_by: a lambda expression to use to sort the values. :return: the sorted values. """ return sorted(self._data.values(), key=sort_by) @@ -217,6 +218,30 @@ def save(self, open=open): os.chmod(self._real_path, 0o600) +class ServerData: + def __init__( + self, + name: str, + target: str, + url: str, + from_store: bool, + api_key: typing.Optional[str] = None, + insecure: typing.Optional[bool] = None, + ca_data: typing.Optional[str] = None, + token: typing.Optional[str] = None, + secret: typing.Optional[str] = None, + ): + self.name = name + self.target = target + self.url = url + self.from_store = from_store + self.api_key = api_key + self.insecure = insecure + self.ca_data = ca_data + self.token = token + self.secret = secret + + class ServerStore(DataStore): """Defines a metadata store for server information. @@ -252,26 +277,29 @@ def get_all_servers(self): """ return self._get_sorted_values(lambda s: s["name"]) - def set(self, name, url, api_key, insecure=False, ca_data=None): + def set(self, name, target, url, api_key=None, insecure=False, ca_data=None, token=None, secret=None): """ Add (or update) information about a Connect server :param name: the nickname for the Connect server. + :param target: the type of target (connect/shinyapps). :param url: the full URL for the Connect server. :param api_key: the API key to use to authenticate with the Connect server. :param insecure: a flag to disable TLS verification. :param ca_data: client side certificate data to use for TLS. + :param token: shinyapps.io token. + :param token: shinyapps.io secret. """ - self._set( - name, - dict( - name=name, - url=url, - api_key=api_key, - insecure=insecure, - ca_cert=ca_data, - ), + common_data = dict( + target=target, + name=name, + url=url, ) + if target == "connect": + target_data = dict(api_key=api_key, insecure=insecure, ca_cert=ca_data) + else: + target_data = dict(token=token, secret=secret) + self._set(name, {**common_data, **target_data}) def remove_by_name(self, name): """ @@ -289,7 +317,7 @@ def remove_by_url(self, url): """ return self._remove_by_value_attr("name", "url", url) - def resolve(self, name, url, api_key, insecure, ca_data): + def resolve(self, name, url, api_key, insecure, ca_data, target="connect"): """ This function will resolve the given inputs into a set of server information. It assumes that either `name` or `url` is provided. @@ -327,15 +355,28 @@ def resolve(self, name, url, api_key, insecure, ca_data): entry = None if entry: - return ( + return ServerData( + name, + entry["target"], entry["url"], - entry["api_key"], - entry["insecure"], - entry["ca_cert"], True, + insecure=entry.get("insecure"), + ca_data=entry.get("ca_cert"), + api_key=entry.get("api_key"), + token=entry.get("token"), + secret=entry.get("secret"), ) else: - return url, api_key, insecure, ca_data, False + return ServerData( + name, + # TODO (mslynch): this function needs to receive target + "connect", + url, + False, + insecure=insecure, + ca_data=ca_data, + api_key=api_key, + ) def sha1(s): diff --git a/tests/test_api.py b/tests/test_api.py index ec12637d..c3e29c76 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1,6 +1,6 @@ from unittest import TestCase -from rsconnect.api import RSConnect +from rsconnect.api import RSConnectClient class TestAPI(TestCase): @@ -14,12 +14,12 @@ def test_output_task_log(self): } output = [] - self.assertEqual(RSConnect.output_task_log(task_status, 0, output.append), 3) + self.assertEqual(RSConnectClient.output_task_log(task_status, 0, output.append), 3) self.assertEqual(lines, output) task_status["last_status"] = 4 task_status["status"] = ["line 4"] - self.assertEqual(RSConnect.output_task_log(task_status, 3, output.append), 4) + self.assertEqual(RSConnectClient.output_task_log(task_status, 3, output.append), 4) self.assertEqual(len(output), 4) self.assertEqual(output[3], "line 4") diff --git a/tests/test_main.py b/tests/test_main.py index 9ebc2319..b5c198af 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -21,7 +21,7 @@ class TestMain(TestCase): def test_validate_deploy_to_args(self): - server_store.set("fake", "http://example.com", None) + server_store.set("fake", "connect", "http://example.com", None) try: with self.assertRaises(RSConnectException): @@ -113,3 +113,47 @@ def test_deploy_api(self): result = runner.invoke(cli, args) self.assertEqual(result.exit_code, 0, result.output) self.assertIn("OK", result.output) + + def test_add_connect(self): + connect_server = self.require_connect() + api_key = self.require_api_key() + runner = CliRunner() + result = runner.invoke(cli, ["add", "--name", "my-connect", "--server", connect_server, "--api-key", api_key]) + self.assertEqual(result.exit_code, 0, result.output) + self.assertIn("OK", result.output) + + def test_add_shinyapps(self): + runner = CliRunner() + result = runner.invoke( + cli, + [ + "add", + "--target", + "shinyapps", + "--name", + "my-shinyapps", + "--token", + "someToken", + "--secret", + "c29tZVNlY3JldAo=", + ], + ) + self.assertEqual(result.exit_code, 0, result.output) + self.assertIn("shinyapps.io credential", result.output) + + def test_add_shinyapps_missing_options(self): + runner = CliRunner() + result = runner.invoke( + cli, + [ + "add", + "--target", + "shinyapps", + "--name", + "my-shinyapps", + "--token", + "someToken", + ], + ) + self.assertEqual(result.exit_code, 1, result.output) + self.assertEqual(str(result.exception), "API key must be specified when using target type 'shinyapps'.") diff --git a/tests/test_metadata.py b/tests/test_metadata.py index ffcefe48..b4f60ceb 100644 --- a/tests/test_metadata.py +++ b/tests/test_metadata.py @@ -17,9 +17,10 @@ def setUp(self): self.server_store_path = join(self.tempDir, "servers.json") self.assertFalse(exists(self.server_store_path)) - self.server_store.set("foo", "http://connect.local", "notReallyAnApiKey", ca_data="/certs/connect") - self.server_store.set("bar", "http://connect.remote", "differentApiKey", insecure=True) - self.assertEqual(len(self.server_store.get_all_servers()), 2, "Unexpected servers after setup") + self.server_store.set("foo", "connect", "http://connect.local", "notReallyAnApiKey", ca_data="/certs/connect") + self.server_store.set("bar", "connect", "http://connect.remote", "differentApiKey", insecure=True) + self.server_store.set("baz", "shinyapps", "https://shinyapps.io", token="someToken", secret="c29tZVNlY3JldAo=") + self.assertEqual(len(self.server_store.get_all_servers()), 3, "Unexpected servers after setup") def tearDown(self): # clean up our temp test directory created with tempfile.mkdtemp() @@ -29,6 +30,7 @@ def test_add(self): self.assertEqual( self.server_store.get_by_name("foo"), dict( + target="connect", name="foo", url="http://connect.local", api_key="notReallyAnApiKey", @@ -40,6 +42,7 @@ def test_add(self): self.assertEqual( self.server_store.get_by_name("bar"), dict( + target="connect", name="bar", url="http://connect.remote", api_key="differentApiKey", @@ -48,6 +51,17 @@ def test_add(self): ), ) + self.assertEqual( + self.server_store.get_by_name("baz"), + dict( + target="shinyapps", + name="baz", + url="https://shinyapps.io", + token="someToken", + secret="c29tZVNlY3JldAo=", + ), + ) + def test_remove_by_name(self): self.server_store.remove_by_name("foo") self.assertIsNone(self.server_store.get_by_name("foo")) @@ -64,28 +78,30 @@ def test_remove_by_url(self): def test_remove_not_found(self): self.assertFalse(self.server_store.remove_by_name("frazzle")) - self.assertEqual(len(self.server_store.get_all_servers()), 2) + self.assertEqual(len(self.server_store.get_all_servers()), 3) self.assertFalse(self.server_store.remove_by_url("http://frazzle")) - self.assertEqual(len(self.server_store.get_all_servers()), 2) + self.assertEqual(len(self.server_store.get_all_servers()), 3) def test_list(self): servers = self.server_store.get_all_servers() - self.assertEqual(len(servers), 2) + self.assertEqual(len(servers), 3) self.assertEqual(servers[0]["name"], "bar") self.assertEqual(servers[0]["url"], "http://connect.remote") - self.assertEqual(servers[1]["name"], "foo") - self.assertEqual(servers[1]["url"], "http://connect.local") + self.assertEqual(servers[1]["name"], "baz") + self.assertEqual(servers[1]["url"], "https://shinyapps.io") + self.assertEqual(servers[2]["name"], "foo") + self.assertEqual(servers[2]["url"], "http://connect.local") def check_resolve_call(self, name, server, api_key, insecure, ca_cert, should_be_from_store): - server, api_key, insecure, ca_cert, from_store = self.server_store.resolve( + server_data = self.server_store.resolve( name, server, api_key, insecure, ca_cert ) - self.assertEqual(server, "http://connect.local") - self.assertEqual(api_key, "notReallyAnApiKey") - self.assertEqual(insecure, False) - self.assertEqual(ca_cert, "/certs/connect") - self.assertTrue(from_store, should_be_from_store) + self.assertEqual(server_data.url, "http://connect.local") + self.assertEqual(server_data.api_key, "notReallyAnApiKey") + self.assertEqual(server_data.insecure, False) + self.assertEqual(server_data.ca_data, "/certs/connect") + self.assertTrue(server_data.from_store, should_be_from_store) def test_resolve_by_name(self): self.check_resolve_call("foo", None, None, None, None, True) @@ -96,11 +112,12 @@ def test_resolve_by_url(self): def test_resolve_by_default(self): # with multiple entries, server None will not resolve by default name, server, api_key, insecure, ca_cert = None, None, None, None, None - server, api_key, insecure, ca_cert, _ = self.server_store.resolve(name, server, api_key, insecure, ca_cert) - self.assertEqual(server, None) + server_data = self.server_store.resolve(name, server, api_key, insecure, ca_cert) + self.assertEqual(server_data.url, None) # with only a single entry, server None will resolve to that entry self.server_store.remove_by_url("http://connect.remote") + self.server_store.remove_by_url("https://shinyapps.io") self.check_resolve_call(None, None, None, None, None, True) def test_resolve_from_args(self): @@ -111,15 +128,15 @@ def test_resolve_from_args(self): True, "fake-cert", ) - server, api_key, insecure, ca_cert, from_store = self.server_store.resolve( + server_data = self.server_store.resolve( name, server, api_key, insecure, ca_cert ) - self.assertEqual(server, "https://secured.connect") - self.assertEqual(api_key, "an-api-key") - self.assertTrue(insecure) - self.assertEqual(ca_cert, "fake-cert") - self.assertFalse(from_store) + self.assertEqual(server_data.url, "https://secured.connect") + self.assertEqual(server_data.api_key, "an-api-key") + self.assertTrue(server_data.insecure) + self.assertEqual(server_data.ca_data, "fake-cert") + self.assertFalse(server_data.from_store) def test_save_and_load(self): temp = tempfile.mkdtemp() @@ -128,7 +145,7 @@ def test_save_and_load(self): self.assertFalse(exists(path)) - server_store.set("foo", "http://connect.local", "notReallyAnApiKey", ca_data="/certs/connect") + server_store.set("foo", "connect", "http://connect.local", "notReallyAnApiKey", ca_data="/certs/connect") self.assertTrue(exists(path)) From 7173bd0dffd60f38684626b3620bcddf3c2fee79 Mon Sep 17 00:00:00 2001 From: Matthew Lynch Date: Mon, 27 Jun 2022 08:40:42 -0500 Subject: [PATCH 02/57] clearer date formatting for shinyapps.io signatures --- rsconnect/api.py | 17 +---------------- rsconnect/main.py | 2 +- 2 files changed, 2 insertions(+), 17 deletions(-) diff --git a/rsconnect/api.py b/rsconnect/api.py index 06046964..db480a37 100644 --- a/rsconnect/api.py +++ b/rsconnect/api.py @@ -340,19 +340,7 @@ def _get_canonical_request_signature(self, request): def get_extra_headers(self, url, method, body): canonical_request_method = method.upper() canonical_request_path = parse.urlparse(url).path - - # TODO (mslynch): there has got to be a better way than this - timetuple = datetime.datetime.utcnow().utctimetuple() - canonical_request_date = "%s, %02d %s %04d %02d:%02d:%02d %s" % ( - ["Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"][timetuple[6]], - timetuple[2], - ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"][timetuple[1] - 1], - timetuple[0], - timetuple[3], - timetuple[4], - timetuple[5], - "GMT", - ) + canonical_request_date = datetime.datetime.utcnow().strftime('%a, %d %b %Y %H:%M:%S GMT') # get request checksum md5 = hashlib.md5() @@ -385,9 +373,6 @@ def create_application(self, account_id, application_name): def get_accounts(self): return self.get("/v1/accounts/") - # def get_current_user(self): - # return self.get('v1/users/me') - def create_bundle(self, application_id: int, content_type: str, content_length: int, checksum: str): bundle_data = { "application": application_id, diff --git a/rsconnect/main.py b/rsconnect/main.py index 8cd7e1c4..a7714ffd 100644 --- a/rsconnect/main.py +++ b/rsconnect/main.py @@ -333,7 +333,7 @@ def add(name, target, server, api_key, insecure, cacert, token, secret, verbose) click.echo('Added Connect server "%s" with URL %s' % (name, real_server.url)) else: shinyapps_server = api.ShinyappsServer(server or "https://api.shinyapps.io", name, token, secret) - # TODO (mslynch): check credential and account here + # TODO (mslynch): test credential and account against server here server_store.set( name, target, shinyapps_server.url, token=shinyapps_server.token, secret=shinyapps_server.secret ) From 983b8245f1624332ca7b8a1ad0fcdbbc9acdf16b Mon Sep 17 00:00:00 2001 From: Matthew Lynch Date: Mon, 27 Jun 2022 11:06:43 -0500 Subject: [PATCH 03/57] better S3 error handling and shinyapps server connection/auth check --- rsconnect/actions.py | 9 +++++++++ rsconnect/api.py | 39 ++++++++++++++++++++++++--------------- rsconnect/main.py | 20 ++++++++++++++++---- 3 files changed, 49 insertions(+), 19 deletions(-) diff --git a/rsconnect/actions.py b/rsconnect/actions.py index 79a1a3d2..923de397 100644 --- a/rsconnect/actions.py +++ b/rsconnect/actions.py @@ -212,6 +212,15 @@ def test_server(connect_server): raise api.RSConnectException("\n".join(failures)) +def test_shinyapps_server(server: api.ShinyappsServer): + with api.ShinyappsClient(server) as client: + try: + result = client.get_current_user() + server.handle_bad_response(result) + except api.RSConnectException as exc: + raise api.RSConnectException(f"Failed to verify with shinyapps.io ({str(exc)}).") + + def test_api_key(connect_server): """ Test that an API Key may be used to authenticate with the given RStudio Connect server. diff --git a/rsconnect/api.py b/rsconnect/api.py index db480a37..13980686 100644 --- a/rsconnect/api.py +++ b/rsconnect/api.py @@ -93,6 +93,13 @@ def __init__(self, url, api_key, insecure=False, ca_data=None): self.cookie_jar = CookieJar() +class S3Server(AbstractRemoteServer): + remote_name = 'S3' + + def __init__(self, url: str): + self.url = url + + RemoteServer = typing.Union[ShinyappsServer, RSConnectServer] @@ -243,7 +250,7 @@ def get_content(self, content_guid): return results def wait_for_task( - self, task_id, log_callback, abort_func=lambda: False, timeout=None, poll_wait=0.5, raise_on_error=True + self, task_id, log_callback, abort_func=lambda: False, timeout=None, poll_wait=0.5, raise_on_error=True ): last_status = None @@ -391,6 +398,9 @@ def deploy_application(self, bundle_id, app_id): def get_task(self, task_id): return self.get(f"/v1/tasks/{task_id}", query_params={"legacy": "true"}) + def get_current_user(self): + return self.get('/v1/users/me') + def wait_until_task_is_successful(self, task_id, timeout=60): counter = 1 status = None @@ -426,8 +436,6 @@ def prepare_deploy(self, app_id, app_name, app_title, title_is_default, bundle_s self._server.handle_bad_response(application) bundle = self.create_bundle(application.json_data["id"], "application/x-tar", bundle_size, bundle_hash) - - # TODO (mslynch) _server isn't meant to handle S3 errors self._server.handle_bad_response(bundle) return {"app_id": application.json_data["id"], "app_url": application.json_data["url"], **bundle.json_data} @@ -548,18 +556,19 @@ def do_bundle_deploy(remote_server: RemoteServer, app_id, name, title, title_is_ app_id, name, title, title_is_default, bundle_size, bundle_hash, env_vars ) - parsed_upload_url = urlparse(prepare_deploy_result["presigned_url"]) + upload_url = prepare_deploy_result["presigned_url"] + parsed_upload_url = urlparse(upload_url) with S3Client(f"{parsed_upload_url.scheme}://{parsed_upload_url.netloc}", timeout=120) as client: upload_result = client.upload( - prepare_deploy_result["presigned_url"], + upload_url, prepare_deploy_result["presigned_checksum"], bundle_size, contents, ) - remote_server.handle_bad_response(upload_result) + S3Server(upload_url).handle_bad_response(upload_result) with ShinyappsClient(remote_server, timeout=120) as client: - deploy_result = client.do_deploy(prepare_deploy_result["id"], prepare_deploy_result["app_id"]) + client.do_deploy(prepare_deploy_result["id"], prepare_deploy_result["app_id"]) webbrowser.open_new(prepare_deploy_result["app_url"]) @@ -567,14 +576,14 @@ def do_bundle_deploy(remote_server: RemoteServer, app_id, name, title, title_is_ def emit_task_log( - connect_server, - app_id, - task_id, - log_callback, - abort_func=lambda: False, - timeout=None, - poll_wait=0.5, - raise_on_error=True, + connect_server, + app_id, + task_id, + log_callback, + abort_func=lambda: False, + timeout=None, + poll_wait=0.5, + raise_on_error=True, ): """ Helper for spooling the deployment log for an app. diff --git a/rsconnect/main.py b/rsconnect/main.py index a7714ffd..367cda1c 100644 --- a/rsconnect/main.py +++ b/rsconnect/main.py @@ -47,7 +47,7 @@ write_environment_file, write_notebook_manifest_json, write_quarto_manifest_json, - fake_module_file_from_directory, + fake_module_file_from_directory, test_shinyapps_server, ) from .actions_content import ( download_bundle, @@ -223,6 +223,10 @@ def _test_server_and_api(server, api_key, insecure, ca_cert): return real_server, me +def _test_shinyappsio_creds(server: api.ShinyappsServer): + with cli_feedback("Checking shinyapps.io credential"): + test_shinyapps_server(server) + # noinspection SpellCheckingInspection @cli.command( @@ -333,7 +337,8 @@ def add(name, target, server, api_key, insecure, cacert, token, secret, verbose) click.echo('Added Connect server "%s" with URL %s' % (name, real_server.url)) else: shinyapps_server = api.ShinyappsServer(server or "https://api.shinyapps.io", name, token, secret) - # TODO (mslynch): test credential and account against server here + _test_shinyappsio_creds(shinyapps_server) + server_store.set( name, target, shinyapps_server.url, token=shinyapps_server.token, secret=shinyapps_server.secret ) @@ -518,6 +523,7 @@ def deploy(): def _validate_deploy_to_args(name, url, api_key, insecure, ca_cert, api_key_is_required=True) -> api.RemoteServer: + # TODO (mslynch): accept non-saved credentials here (maybe)? """ Validate that the user gave us enough information to talk to a Connect server. @@ -563,9 +569,15 @@ def _validate_deploy_to_args(name, url, api_key, insecure, ca_cert, api_key_is_r return connect_server else: - # TODO (mslynch): validate shinyapps data # TODO (mslynch): replace nickname with account name - return api.ShinyappsServer(server_data.url, server_data.name, server_data.token, server_data.secret) + if not server_data.token: + raise api.RSConnectException("A token must be specified.") + if not server_data.token: + raise api.RSConnectException("A secret must be specified.") + + server = api.ShinyappsServer(server_data.url, server_data.name, server_data.token, server_data.secret) + test_shinyapps_server(server) + return server def _warn_on_ignored_manifest(directory): From eabc315e3eaa833b9ec249270ab5d5fcd28cbe14 Mon Sep 17 00:00:00 2001 From: Matthew Lynch Date: Tue, 28 Jun 2022 10:31:26 -0500 Subject: [PATCH 04/57] fix refactor merge issues and tests --- rsconnect/api.py | 204 +++++++++++++++++++++++++++-------------- rsconnect/main.py | 25 ++--- tests/test_api.py | 2 +- tests/test_main.py | 3 + tests/test_metadata.py | 8 +- 5 files changed, 156 insertions(+), 86 deletions(-) diff --git a/rsconnect/api.py b/rsconnect/api.py index 2e6ad5ad..a7a550fc 100644 --- a/rsconnect/api.py +++ b/rsconnect/api.py @@ -55,7 +55,11 @@ def handle_bad_response(self, response): # also catches all error conditions which we will report as "not running Connect". else: if response.json_data and "error" in response.json_data and response.json_data["error"] is not None: - error = "%s reported an error: %s" % (self.remote_name, response.json_data["error"]) + error = "%s reported an error (calling %s): %s" % ( + self.remote_name, + response.full_uri, + response.json_data["error"], + ) raise RSConnectException(error) if response.status < 200 or response.status > 299: raise RSConnectException( @@ -97,7 +101,7 @@ def __init__(self, url, api_key, insecure=False, ca_data=None): class S3Server(AbstractRemoteServer): - remote_name = 'S3' + remote_name = "S3" def __init__(self, url: str): self.url = url @@ -253,7 +257,7 @@ def get_content(self, content_guid): return results def wait_for_task( - self, task_id, log_callback, abort_func=lambda: False, timeout=None, poll_wait=0.5, raise_on_error=True + self, task_id, log_callback, abort_func=lambda: False, timeout=None, poll_wait=0.5, raise_on_error=True ): last_status = None @@ -333,13 +337,15 @@ def __init__( cacert: IO = None, ca_data: str = None, cookies=None, + token: str = None, + secret: str = None, timeout: int = 30, logger=console_logger, **kwargs ) -> None: self.reset() self._d = kwargs - self.setup_connect_server(name, url or kwargs.get("server"), api_key, insecure, cacert, ca_data) + self.setup_remote_server(name, url or kwargs.get("server"), api_key, insecure, cacert, ca_data) self.setup_client(cookies, timeout) self.logger = logger @@ -355,7 +361,7 @@ def fromConnectServer(cls, connect_server, **kwargs): def reset(self): self._d = None - self.connect_server = None + self.remote_server = None self.client = None self.logger = None gc.collect() @@ -366,7 +372,7 @@ def drop_context(self): gc.collect() return self - def setup_connect_server( + def setup_remote_server( self, name: str = None, url: str = None, @@ -384,13 +390,21 @@ def setup_connect_server( ca_data = text_type(cacert.read()) server_data = ServerStore().resolve(name, url, api_key, insecure, ca_data) - self.connect_server = RSConnectServer(server_data.url, - server_data.api_key, - server_data.insecure, - server_data.ca_data) + + if server_data.api_key is not None: + self.remote_server = RSConnectServer( + server_data.url, server_data.api_key, server_data.insecure, server_data.ca_data + ) + else: + self.remote_server = ShinyappsServer( + server_data.url, server_data.name, server_data.token, server_data.secret + ) def setup_client(self, cookies=None, timeout=30, **kwargs): - self.client = RSConnectClient(self.connect_server, cookies, timeout) + if isinstance(self.remote_server, RSConnectServer): + self.client = RSConnectClient(self.remote_server, cookies, timeout) + else: + self.client = ShinyappsClient(self.remote_server) @property def state(self): @@ -411,10 +425,12 @@ def validate_server( insecure: bool = False, cacert: IO = None, api_key_is_required: bool = False, + token: str = None, + secret: str = None, **kwargs ): """ - Validate that the user gave us enough information to talk to a Connect server. + Validate that the user gave us enough information to talk to shinyapps.io or a Connect server. :param name: the nickname, if any, specified by the user. :param url: the URL, if any, specified by the user. @@ -423,26 +439,31 @@ def validate_server( :param cacert: the file object of a CA certs file containing certificates to use. :param api_key_is_required: a flag that notes whether the API key is required or may be omitted. + :param token: The shinyapps.io authentication token. + :param secret: The shinyapps.io authentication secret. """ - url = url or self.connect_server.url - api_key = api_key or self.connect_server.api_key - insecure = insecure or self.connect_server.insecure - api_key_is_required = api_key_is_required or self.get("api_key_is_required", **kwargs) - server_store = ServerStore() - + url = url or self.remote_server.url + ca_data = None if cacert: ca_data = text_type(cacert.read()) + if isinstance(self.remote_server, RSConnectServer): + api_key = api_key or self.remote_server.api_key + insecure = insecure or self.remote_server.insecure + if not ca_data: + ca_data = self.remote_server.ca_data else: - ca_data = self.connect_server.ca_data + token = token or self.remote_server.token + secret = secret or self.remote_server.secret + + api_key_is_required = api_key_is_required or self.get("api_key_is_required", **kwargs) + server_store = ServerStore() if name and url: raise RSConnectException("You must specify only one of -n/--name or -s/--server, not both") if not name and not url: raise RSConnectException("You must specify one of -n/--name or -s/--server.") - server_data = server_store.resolve( - name, url, api_key, insecure, ca_data - ) + server_data = server_store.resolve(name, url, api_key, insecure, ca_data) # This can happen if the user specifies neither --name or --server and there's not # a single default to go with. @@ -453,7 +474,7 @@ def validate_server( # If our info came from the command line, make sure the URL really works. if not server_data.from_store: - self.server_settings + self.server_settings() connect_server.api_key = api_key @@ -466,8 +487,8 @@ def validate_server( if not server_data.from_store: _ = self.verify_api_key() - self.connect_server = connect_server - self.client = RSConnectClient(self.connect_server) + self.remote_server = connect_server + self.client = RSConnectClient(self.remote_server) return self @@ -520,6 +541,10 @@ def check_server_capabilities(self, capability_functions): :param details_source: the source for obtaining server details, gather_server_details(), by default. """ + # TODO (mslynch): check shinyapps capabilities + if isinstance(self.remote_server, ShinyappsServer): + return self + details = self.server_details for function in capability_functions: @@ -542,17 +567,62 @@ def deploy_bundle( bundle: IO = None, env_vars=None, ): - result = self.client.deploy( - app_id or self.get("app_id"), - deployment_name or self.get("deployment_name"), - title or self.get("title"), - title_is_default or self.get("title_is_default"), - bundle or self.get("bundle"), - env_vars or self.get("env_vars"), - ) - self.connect_server.handle_bad_response(result) - self.state["deployed_info"] = result - return self + app_id = app_id or self.get("app_id") + deployment_name = deployment_name or self.get("deployment_name") + title = title or self.get("title") + title_is_default = title_is_default or self.get("title_is_default") + bundle = bundle or self.get("bundle") + env_vars = env_vars or self.get("env_vars") + + if isinstance(self.remote_server, RSConnectServer): + result = self.client.deploy( + app_id, + deployment_name, + title, + title_is_default, + bundle, + env_vars, + ) + self.remote_server.handle_bad_response(result) + self.state["deployed_info"] = result + return self + else: + contents = bundle.read() + bundle_size = len(contents) + bundle_hash = hashlib.md5(contents).hexdigest() + + prepare_deploy_result = self.client.prepare_deploy( + app_id, + deployment_name, + title, + title_is_default, + bundle_size, + bundle_hash, + env_vars, + ) + + upload_url = prepare_deploy_result["presigned_url"] + parsed_upload_url = urlparse(upload_url) + with S3Client(f"{parsed_upload_url.scheme}://{parsed_upload_url.netloc}", timeout=120) as s3_client: + upload_result = s3_client.upload( + f"{parsed_upload_url.path}?{parsed_upload_url.query}", + prepare_deploy_result["presigned_checksum"], + bundle_size, + contents, + ) + S3Server(upload_url).handle_bad_response(upload_result) + + self.client.do_deploy(prepare_deploy_result["id"], prepare_deploy_result["app_id"]) + + webbrowser.open_new(prepare_deploy_result["app_url"]) + + self.state["deployed_info"] = { + "app_url": prepare_deploy_result["app_url"], + "app_id": prepare_deploy_result["id"], + "app_guid": None, + "title": title, + } + return self def emit_task_log( self, @@ -567,7 +637,6 @@ def emit_task_log( """ Helper for spooling the deployment log for an app. - :param connect_server: the Connect server information. :param app_id: the ID of the app that was deployed. :param task_id: the ID of the task that is tracking the deployment of the app.. :param log_callback: the callback to use to write the log to. If this is None @@ -579,20 +648,21 @@ def emit_task_log( :param raise_on_error: whether to raise an exception when a task is failed, otherwise we return the task_result so we can record the exit code. """ - app_id = app_id or self.state["deployed_info"]["app_id"] - task_id = task_id or self.state["deployed_info"]["task_id"] - log_lines, _ = self.client.wait_for_task( - task_id, log_callback.info, abort_func, timeout, poll_wait, raise_on_error - ) - self.connect_server.handle_bad_response(log_lines) - app_config = self.client.app_config(app_id) - self.connect_server.handle_bad_response(app_config) - app_dashboard_url = app_config.get("config_url") - log_callback.info("Deployment completed successfully.") - log_callback.info("\t Dashboard content URL: ") - log_callback.debug(app_dashboard_url) - log_callback.info("\t Direct content URL: ") - log_callback.debug(self.state["deployed_info"]["app_url"]) + if isinstance(self.remote_server, RSConnectServer): + app_id = app_id or self.state["deployed_info"]["app_id"] + task_id = task_id or self.state["deployed_info"]["task_id"] + log_lines, _ = self.client.wait_for_task( + task_id, log_callback.info, abort_func, timeout, poll_wait, raise_on_error + ) + self.remote_server.handle_bad_response(log_lines) + app_config = self.client.app_config(app_id) + self.remote_server.handle_bad_response(app_config) + app_dashboard_url = app_config.get("config_url") + log_callback.info("Deployment completed successfully.") + log_callback.info("\t Dashboard content URL: ") + log_callback.debug(app_dashboard_url) + log_callback.info("\t Direct content URL: ") + log_callback.debug(self.state["deployed_info"]["app_url"]) return self @@ -609,7 +679,7 @@ def save_deployed_info(self, *args, **kwargs): deployed_info = self.get("deployed_info", *args, **kwargs) app_store.set( - self.connect_server.url, + self.remote_server.url, abspath(path), deployed_info["app_url"], deployed_info["app_id"], @@ -622,7 +692,7 @@ def save_deployed_info(self, *args, **kwargs): @cls_logged("Validating app mode...") def validate_app_mode(self, *args, **kwargs): - connect_server = self.connect_server + connect_server = self.remote_server path = ( self.get("path", **kwargs) or self.get("file", **kwargs) @@ -669,7 +739,7 @@ def validate_app_mode(self, *args, **kwargs): def server_settings(self): try: result = self.client.server_settings() - self.connect_server.handle_bad_response(result) + self.remote_server.handle_bad_response(result) except SSLError as ssl_error: raise RSConnectException("There is an SSL/TLS configuration problem: %s" % ssl_error) return result @@ -689,7 +759,7 @@ def verify_api_key(self): @property def api_username(self): result = self.client.me() - self.connect_server.handle_bad_response(result) + self.remote_server.handle_bad_response(result) return result["username"] @property @@ -701,7 +771,7 @@ def python_info(self): :return: the Python installation information from Connect. """ result = self.client.python_settings() - self.connect_server.handle_bad_response(result) + self.remote_server.handle_bad_response(result) return result @property @@ -761,7 +831,7 @@ def make_deployment_name(self, title, force_unique): # Now, make sure it's unique, if needed. if force_unique: - name = find_unique_name(self.connect_server, name) + name = find_unique_name(self.remote_server, name) return name @@ -799,7 +869,7 @@ def _get_canonical_request_signature(self, request): def get_extra_headers(self, url, method, body): canonical_request_method = method.upper() canonical_request_path = parse.urlparse(url).path - canonical_request_date = datetime.datetime.utcnow().strftime('%a, %d %b %Y %H:%M:%S GMT') + canonical_request_date = datetime.datetime.utcnow().strftime("%a, %d %b %Y %H:%M:%S GMT") # get request checksum md5 = hashlib.md5() @@ -851,7 +921,7 @@ def get_task(self, task_id): return self.get(f"/v1/tasks/{task_id}", query_params={"legacy": "true"}) def get_current_user(self): - return self.get('/v1/users/me') + return self.get("/v1/users/me") def wait_until_task_is_successful(self, task_id, timeout=60): counter = 1 @@ -1033,14 +1103,14 @@ def do_bundle_deploy(remote_server: RemoteServer, app_id, name, title, title_is_ def emit_task_log( - connect_server, - app_id, - task_id, - log_callback, - abort_func=lambda: False, - timeout=None, - poll_wait=0.5, - raise_on_error=True, + connect_server, + app_id, + task_id, + log_callback, + abort_func=lambda: False, + timeout=None, + poll_wait=0.5, + raise_on_error=True, ): """ Helper for spooling the deployment log for an app. diff --git a/rsconnect/main.py b/rsconnect/main.py index 4f1711f6..15a913ba 100644 --- a/rsconnect/main.py +++ b/rsconnect/main.py @@ -243,6 +243,7 @@ def _test_server_and_api(server, api_key, insecure, ca_cert): return real_server, me + def _test_shinyappsio_creds(server: api.ShinyappsServer): with cli_feedback("Checking shinyapps.io credential"): test_shinyapps_server(server) @@ -413,9 +414,9 @@ def details(name, server, api_key, insecure, cacert, verbose): ce = RSConnectExecutor(name, server, api_key, insecure, cacert).validate_server() - click.echo(" RStudio Connect URL: %s" % ce.connect_server.url) + click.echo(" RStudio Connect URL: %s" % ce.remote_server.url) - if not ce.connect_server.api_key: + if not ce.remote_server.api_key: return with cli_feedback("Gathering details"): @@ -565,7 +566,7 @@ def _validate_deploy_to_args(name, url, api_key, insecure, ca_cert, api_key_is_r # This can happen if the user specifies neither --name or --server and there's not # a single default to go with. - if not server_data.real_server: + if not server_data.url: raise api.RSConnectException("You must specify one of -n/--name or -s/--server.") if server_data.target == "connect": @@ -1675,7 +1676,7 @@ def content_search( with cli_feedback("", stderr=True): ce = RSConnectExecutor(name, server, api_key, insecure, cacert, logger=None).validate_server() result = search_content( - ce.connect_server, published, unpublished, content_type, r_version, py_version, title_contains, order_by + ce.remote_server, published, unpublished, content_type, r_version, py_version, title_contains, order_by ) json.dump(result, sys.stdout, indent=2) @@ -1727,7 +1728,7 @@ def content_describe(name, server, api_key, insecure, cacert, guid, verbose): set_verbosity(verbose) with cli_feedback("", stderr=True): ce = RSConnectExecutor(name, server, api_key, insecure, cacert, logger=None).validate_server() - result = get_content(ce.connect_server, guid) + result = get_content(ce.remote_server, guid) json.dump(result, sys.stdout, indent=2) @@ -1791,7 +1792,7 @@ def content_bundle_download(name, server, api_key, insecure, cacert, guid, outpu if exists(output) and not overwrite: raise RSConnectException("The output file already exists: %s" % output) - result = download_bundle(ce.connect_server, guid) + result = download_bundle(ce.remote_server, guid) with open(output, "wb") as f: f.write(result.response_body) @@ -1846,7 +1847,7 @@ def add_content_build(name, server, api_key, insecure, cacert, guid, verbose): set_verbosity(verbose) with cli_feedback("", stderr=True): ce = RSConnectExecutor(name, server, api_key, insecure, cacert, logger=None).validate_server() - build_add_content(ce.connect_server, guid) + build_add_content(ce.remote_server, guid) if len(guid) == 1: logger.info('Added "%s".' % guid[0]) else: @@ -1911,7 +1912,7 @@ def remove_content_build(name, server, api_key, insecure, cacert, guid, all, pur with cli_feedback("", stderr=True): ce = RSConnectExecutor(name, server, api_key, insecure, cacert, logger=None).validate_server() _validate_build_rm_args(guid, all, purge) - guids = build_remove_content(ce.connect_server, guid, all, purge) + guids = build_remove_content(ce.remote_server, guid, all, purge) if len(guids) == 1: logger.info('Removed "%s".' % guids[0]) else: @@ -1964,7 +1965,7 @@ def list_content_build(name, server, api_key, insecure, cacert, status, guid, ve set_verbosity(verbose) with cli_feedback("", stderr=True): ce = RSConnectExecutor(name, server, api_key, insecure, cacert, logger=None).validate_server() - result = build_list_content(ce.connect_server, guid, status) + result = build_list_content(ce.remote_server, guid, status) json.dump(result, sys.stdout, indent=2) @@ -2012,7 +2013,7 @@ def get_build_history(name, server, api_key, insecure, cacert, guid, verbose): with cli_feedback("", stderr=True): ce = RSConnectExecutor(name, server, api_key, insecure, cacert) ce.validate_server() - result = build_history(ce.connect_server, guid) + result = build_history(ce.remote_server, guid) json.dump(result, sys.stdout, indent=2) @@ -2075,7 +2076,7 @@ def get_build_logs(name, server, api_key, insecure, cacert, guid, task_id, forma set_verbosity(verbose) with cli_feedback("", stderr=True): ce = RSConnectExecutor(name, server, api_key, insecure, cacert, logger=None).validate_server() - for line in emit_build_log(ce.connect_server, guid, format, task_id): + for line in emit_build_log(ce.remote_server, guid, format, task_id): sys.stdout.write(line) @@ -2142,7 +2143,7 @@ def start_content_build( logger.set_log_output_format(format) with cli_feedback("", stderr=True): ce = RSConnectExecutor(name, server, api_key, insecure, cacert, logger=None).validate_server() - build_start(ce.connect_server, parallelism, aborted, error, all, poll_wait, debug) + build_start(ce.remote_server, parallelism, aborted, error, all, poll_wait, debug) if __name__ == "__main__": diff --git a/tests/test_api.py b/tests/test_api.py index d621c6da..cbd0d5ba 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -11,7 +11,7 @@ def test_executor_init(self): connect_server = require_connect(self) api_key = require_api_key(self) ce = RSConnectExecutor(None, connect_server, api_key, True, None) - self.assertEqual(ce.connect_server.url, connect_server) + self.assertEqual(ce.remote_server.url, connect_server) def test_output_task_log(self): lines = ["line 1", "line 2", "line 3"] diff --git a/tests/test_main.py b/tests/test_main.py index 3bde5e03..8f2f0b6f 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -1,4 +1,5 @@ import os +import unittest from os.path import join from unittest import TestCase @@ -119,6 +120,8 @@ def test_add_connect(self): self.assertEqual(result.exit_code, 0, result.output) self.assertIn("OK", result.output) + # TODO (mslynch): mock shinyapps.io + @unittest.skip def test_add_shinyapps(self): runner = CliRunner() result = runner.invoke( diff --git a/tests/test_metadata.py b/tests/test_metadata.py index b4f60ceb..10c1bc3d 100644 --- a/tests/test_metadata.py +++ b/tests/test_metadata.py @@ -93,9 +93,7 @@ def test_list(self): self.assertEqual(servers[2]["url"], "http://connect.local") def check_resolve_call(self, name, server, api_key, insecure, ca_cert, should_be_from_store): - server_data = self.server_store.resolve( - name, server, api_key, insecure, ca_cert - ) + server_data = self.server_store.resolve(name, server, api_key, insecure, ca_cert) self.assertEqual(server_data.url, "http://connect.local") self.assertEqual(server_data.api_key, "notReallyAnApiKey") @@ -128,9 +126,7 @@ def test_resolve_from_args(self): True, "fake-cert", ) - server_data = self.server_store.resolve( - name, server, api_key, insecure, ca_cert - ) + server_data = self.server_store.resolve(name, server, api_key, insecure, ca_cert) self.assertEqual(server_data.url, "https://secured.connect") self.assertEqual(server_data.api_key, "an-api-key") From c574b9ba309822ca536ef13de60719997901c4d3 Mon Sep 17 00:00:00 2001 From: Matthew Lynch Date: Tue, 28 Jun 2022 11:57:59 -0500 Subject: [PATCH 05/57] infer target type from `add` options --- rsconnect/main.py | 77 +++++++++++++++++----------------------------- tests/test_main.py | 32 ++++++++++--------- 2 files changed, 45 insertions(+), 64 deletions(-) diff --git a/rsconnect/main.py b/rsconnect/main.py index 15a913ba..b52493f1 100644 --- a/rsconnect/main.py +++ b/rsconnect/main.py @@ -259,14 +259,6 @@ def _test_shinyappsio_creds(server: api.ShinyappsServer): ), ) @click.option("--name", "-n", required=True, help="The nickname of the RStudio Connect server to deploy to.") -@click.option( - "--target", - "-t", - type=click.Choice(["connect", "shinyapps"]), - default="connect", - envvar="RSCONNECT_SERVER_TYPE", - help="The target platform of this credential (Connect or shinyapps.io).", -) @click.option( "--server", "-s", @@ -306,46 +298,44 @@ def _test_shinyappsio_creds(server: api.ShinyappsServer): help="The shinyapps.io token secret.", ) @click.option("--verbose", "-v", is_flag=True, help="Print detailed messages.") -def add(name, target, server, api_key, insecure, cacert, token, secret, verbose): +def add(name, server, api_key, insecure, cacert, token, secret, verbose): set_verbosity(verbose) + connect_options = {"--api-key": api_key, "--insecure": insecure, "--cacert": cacert} + shinyapps_options = {"--token": token, "--secret": secret} + + present_connect_options = [k for k, v in connect_options.items() if v] + present_shinyapps_options = [k for k, v in shinyapps_options.items() if v] + + if present_connect_options and present_shinyapps_options: + raise api.RSConnectException( + f"Connect options ({', '.join(present_connect_options)}) may not be passed alongside " + f"shinyapps.io options ({', '.join(present_shinyapps_options)})." + ) + old_server = server_store.get_by_name(name) - # validate options - if target == "connect": - # # disallowed - # if token is not None: - # raise api.RSConnectException("Token may not be specified when using target type 'connect'.") - # if secret is not None: - # raise api.RSConnectException("Secret may not be specified when using target type 'connect'.") - - # required - if server is None: - raise api.RSConnectException("Server must be specified when using target type 'connect'.") - if api_key is None: - raise api.RSConnectException("API key must be specified when using target type 'connect'.") + if present_shinyapps_options: + if len(present_shinyapps_options) != 2: + raise api.RSConnectException("--token and --secret must both be provided for shinyapps.io.") + + shinyapps_server = api.ShinyappsServer(server or "https://api.shinyapps.io", name, token, secret) + _test_shinyappsio_creds(shinyapps_server) + + server_store.set( + name, 'shinyapps', shinyapps_server.url, token=shinyapps_server.token, secret=shinyapps_server.secret + ) + if old_server: + click.echo('Updated shinyapps.io credential "%s".' % name) + else: + click.echo('Added shinyapps.io credential "%s".' % name) else: - # # disallowed - # if api_key is not None: - # raise api.RSConnectException("API key may not be specified when using target type 'shinyapps'.") - # if insecure is not None: - # raise api.RSConnectException("Insecure may not be specified when using target type 'shinyapps'.") - # if cacert is not None: - # raise api.RSConnectException("CA certificate may not be specified when using target type 'shinyapps'.") - - # required - if token is None: - raise api.RSConnectException("Server must be specified when using target type 'shinyapps'.") - if secret is None: - raise api.RSConnectException("API key must be specified when using target type 'shinyapps'.") - - if target == "connect": # Server must be pingable and the API key must work to be added. real_server, _ = _test_server_and_api(server, api_key, insecure, cacert) server_store.set( name, - target, + 'connect', real_server.url, real_server.api_key, real_server.insecure, @@ -356,17 +346,6 @@ def add(name, target, server, api_key, insecure, cacert, token, secret, verbose) click.echo('Updated Connect server "%s" with URL %s' % (name, real_server.url)) else: click.echo('Added Connect server "%s" with URL %s' % (name, real_server.url)) - else: - shinyapps_server = api.ShinyappsServer(server or "https://api.shinyapps.io", name, token, secret) - _test_shinyappsio_creds(shinyapps_server) - - server_store.set( - name, target, shinyapps_server.url, token=shinyapps_server.token, secret=shinyapps_server.secret - ) - if old_server: - click.echo('Added shinyapps.io credential "%s".' % name) - else: - click.echo('Updated shinyapps.io credential "%s".' % name) @cli.command( diff --git a/tests/test_main.py b/tests/test_main.py index 8f2f0b6f..58406b5d 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -142,18 +142,20 @@ def test_add_shinyapps(self): self.assertIn("shinyapps.io credential", result.output) def test_add_shinyapps_missing_options(self): - runner = CliRunner() - result = runner.invoke( - cli, - [ - "add", - "--target", - "shinyapps", - "--name", - "my-shinyapps", - "--token", - "someToken", - ], - ) - self.assertEqual(result.exit_code, 1, result.output) - self.assertEqual(str(result.exception), "API key must be specified when using target type 'shinyapps'.") + original_api_key_value = os.environ.pop("CONNECT_API_KEY") + try: + runner = CliRunner() + result = runner.invoke( + cli, + [ + "add", + "--name", + "my-shinyapps", + "--token", + "someToken", + ], + ) + self.assertEqual(result.exit_code, 1, result.output) + self.assertEqual(str(result.exception), "--token and --secret must both be provided for shinyapps.io.") + finally: + os.environ["CONNECT_API_KEY"] = original_api_key_value From 0a1e70fcaffaa3d68d03e4db255f5026cb6b57e1 Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Tue, 28 Jun 2022 16:14:05 -0400 Subject: [PATCH 06/57] ca_data needs to fetch ca_data --- rsconnect/metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rsconnect/metadata.py b/rsconnect/metadata.py index e9cad010..4d838409 100644 --- a/rsconnect/metadata.py +++ b/rsconnect/metadata.py @@ -361,7 +361,7 @@ def resolve(self, name, url, api_key, insecure, ca_data, target="connect"): entry["url"], True, insecure=entry.get("insecure"), - ca_data=entry.get("ca_cert"), + ca_data=entry.get("ca_data"), api_key=entry.get("api_key"), token=entry.get("token"), secret=entry.get("secret"), From 5c318c6c55587b1509b0db670f1790d2879a7298 Mon Sep 17 00:00:00 2001 From: Matthew Lynch Date: Wed, 29 Jun 2022 09:53:25 -0500 Subject: [PATCH 07/57] working redeploys for shinyapps --- rsconnect/api.py | 92 +++++++++++++++++++++++++++++++-------------- rsconnect/main.py | 62 +----------------------------- rsconnect/models.py | 16 ++++++++ tests/test_main.py | 18 +-------- 4 files changed, 82 insertions(+), 106 deletions(-) diff --git a/rsconnect/api.py b/rsconnect/api.py index a7a550fc..0e9942f7 100644 --- a/rsconnect/api.py +++ b/rsconnect/api.py @@ -541,7 +541,7 @@ def check_server_capabilities(self, capability_functions): :param details_source: the source for obtaining server details, gather_server_details(), by default. """ - # TODO (mslynch): check shinyapps capabilities + # TODO (mslynch): check shinyapps capabilities? if isinstance(self.remote_server, ShinyappsServer): return self @@ -594,31 +594,28 @@ def deploy_bundle( prepare_deploy_result = self.client.prepare_deploy( app_id, deployment_name, - title, - title_is_default, bundle_size, bundle_hash, - env_vars, ) - upload_url = prepare_deploy_result["presigned_url"] + upload_url = prepare_deploy_result.presigned_url parsed_upload_url = urlparse(upload_url) with S3Client(f"{parsed_upload_url.scheme}://{parsed_upload_url.netloc}", timeout=120) as s3_client: upload_result = s3_client.upload( f"{parsed_upload_url.path}?{parsed_upload_url.query}", - prepare_deploy_result["presigned_checksum"], + prepare_deploy_result.presigned_checksum, bundle_size, contents, ) S3Server(upload_url).handle_bad_response(upload_result) - self.client.do_deploy(prepare_deploy_result["id"], prepare_deploy_result["app_id"]) + self.client.do_deploy(prepare_deploy_result.bundle_id, prepare_deploy_result.app_id) - webbrowser.open_new(prepare_deploy_result["app_url"]) + webbrowser.open_new(prepare_deploy_result.app_url) self.state["deployed_info"] = { - "app_url": prepare_deploy_result["app_url"], - "app_id": prepare_deploy_result["id"], + "app_url": prepare_deploy_result.app_url, + "app_id": prepare_deploy_result.app_id, "app_guid": None, "title": title, } @@ -668,7 +665,7 @@ def emit_task_log( @cls_logged("Saving deployed information...") def save_deployed_info(self, *args, **kwargs): - app_store = self.get("app_store", *args, **kwargs) + app_store: AppStore = self.get("app_store", *args, **kwargs) path = ( self.get("path", **kwargs) or self.get("file", **kwargs) @@ -692,7 +689,6 @@ def save_deployed_info(self, *args, **kwargs): @cls_logged("Validating app mode...") def validate_app_mode(self, *args, **kwargs): - connect_server = self.remote_server path = ( self.get("path", **kwargs) or self.get("file", **kwargs) @@ -716,13 +712,17 @@ def validate_app_mode(self, *args, **kwargs): if app_id is None: # Possible redeployment - check for saved metadata. # Use the saved app information unless overridden by the user. - app_id, existing_app_mode = app_store.resolve(connect_server.url, app_id, app_mode) + app_id, existing_app_mode = app_store.resolve(self.remote_server.url, app_id, app_mode) logger.debug("Using app mode from app %s: %s" % (app_id, app_mode)) elif app_id is not None: # Don't read app metadata if app-id is specified. Instead, we need - # to get this from Connect. - app = get_app_info(connect_server, app_id) - existing_app_mode = AppModes.get_by_ordinal(app.get("app_mode", 0), True) + # to get this from the remote. + if isinstance(self.remote_server, RSConnectServer): + app = get_app_info(self.remote_server, app_id) + existing_app_mode = AppModes.get_by_ordinal(app.get("app_mode", 0), True) + else: + app = get_shinyapp_info(self.remote_server, app_id) + existing_app_mode = AppModes.get_by_cloud_name(app.json_data["mode"]) if existing_app_mode and app_mode != existing_app_mode: msg = ( "Deploying with mode '%s',\n" @@ -852,6 +852,15 @@ def upload(self, path, presigned_checksum, bundle_size, contents): return self.put(path, headers=headers, body=contents, decode_response=False) +class PrepareDeployResult: + def __init__(self, app_id: int, app_url: str, bundle_id: int, presigned_url: str, presigned_checksum: str): + self.app_id = app_id + self.app_url = app_url + self.bundle_id = bundle_id + self.presigned_url = presigned_url + self.presigned_checksum = presigned_checksum + + class ShinyappsClient(HTTPServer): def __init__(self, shinyapps_server: ShinyappsServer, timeout: int = 30): self._token = shinyapps_server.token @@ -891,6 +900,9 @@ def get_extra_headers(self, url, method, body): "X-Content-Checksum": canonical_request_checksum, } + def get_application(self, application_id): + return self.get(f"/v1/applications/{application_id}") + def create_application(self, account_id, application_name): application_data = { "account": account_id, @@ -942,11 +954,11 @@ def wait_until_task_is_successful(self, task_id, timeout=60): counter += 1 click.secho(f"Task done: {description}") - def prepare_deploy(self, app_id, app_name, app_title, title_is_default, bundle_size, bundle_hash, env_vars=None): + def prepare_deploy(self, app_id: typing.Optional[str], app_name: str, bundle_size: int, bundle_hash: str): accounts = self.get_accounts() self._server.handle_bad_response(accounts) account = next( - filter(lambda account: account["name"] == self._server.account_name, accounts.json_data["accounts"]), None + filter(lambda acct: acct["name"] == self._server.account_name, accounts.json_data["accounts"]), None ) # TODO: also check this during `add` command if account is None: @@ -954,13 +966,24 @@ def prepare_deploy(self, app_id, app_name, app_title, title_is_default, bundle_s "No account found by name : %s for given user credential" % self._server.account_name ) - application = self.create_application(account["id"], app_name) + if app_id is None: + application = self.create_application(account["id"], app_name) + else: + application = self.get_application(app_id) self._server.handle_bad_response(application) + app_id = application.json_data["id"] + app_url = application.json_data["url"] - bundle = self.create_bundle(application.json_data["id"], "application/x-tar", bundle_size, bundle_hash) + bundle = self.create_bundle(app_id, "application/x-tar", bundle_size, bundle_hash) self._server.handle_bad_response(bundle) - return {"app_id": application.json_data["id"], "app_url": application.json_data["url"], **bundle.json_data} + return PrepareDeployResult( + int(app_id), + app_url, + int(bundle.json_data["id"]), + bundle.json_data["presigned_url"], + bundle.json_data["presigned_checksum"], + ) def do_deploy(self, bundle_id, app_id): bundle_status_response = self.set_bundle_status(bundle_id, "ready") @@ -1039,6 +1062,13 @@ def get_app_info(connect_server, app_id): return result +def get_shinyapp_info(server, app_id): + with ShinyappsClient(server) as client: + result = client.get_application(app_id) + server.handle_bad_response(result) + return result + + def get_app_config(connect_server, app_id): """ Return the configuration information for an application that has been created @@ -1074,32 +1104,36 @@ def do_bundle_deploy(remote_server: RemoteServer, app_id, name, title, title_is_ remote_server.handle_bad_response(result) return result else: + raise Exception('whuh oh') contents = bundle.read() bundle_size = len(contents) bundle_hash = hashlib.md5(contents).hexdigest() with ShinyappsClient(remote_server, timeout=120) as client: - prepare_deploy_result = client.prepare_deploy( - app_id, name, title, title_is_default, bundle_size, bundle_hash, env_vars - ) + prepare_deploy_result = client.prepare_deploy(app_id, name, bundle_size, bundle_hash) - upload_url = prepare_deploy_result["presigned_url"] + upload_url = prepare_deploy_result.presigned_url parsed_upload_url = urlparse(upload_url) with S3Client(f"{parsed_upload_url.scheme}://{parsed_upload_url.netloc}", timeout=120) as client: upload_result = client.upload( upload_url, - prepare_deploy_result["presigned_checksum"], + prepare_deploy_result.presigned_checksum, bundle_size, contents, ) S3Server(upload_url).handle_bad_response(upload_result) with ShinyappsClient(remote_server, timeout=120) as client: - client.do_deploy(prepare_deploy_result["id"], prepare_deploy_result["app_id"]) + client.do_deploy(prepare_deploy_result.bundle_id, prepare_deploy_result.app_id) - webbrowser.open_new(prepare_deploy_result["app_url"]) + webbrowser.open_new(prepare_deploy_result.app_url) - return {"app_url": prepare_deploy_result["app_url"], "app_id": prepare_deploy_result["id"], "app_guid": None} + return { + "app_url": prepare_deploy_result.app_url, + "app_id": prepare_deploy_result.app_id, + "app_guid": None, + "title": title, + } def emit_task_log( diff --git a/rsconnect/main.py b/rsconnect/main.py index b52493f1..e0961a3b 100644 --- a/rsconnect/main.py +++ b/rsconnect/main.py @@ -323,7 +323,7 @@ def add(name, server, api_key, insecure, cacert, token, secret, verbose): _test_shinyappsio_creds(shinyapps_server) server_store.set( - name, 'shinyapps', shinyapps_server.url, token=shinyapps_server.token, secret=shinyapps_server.secret + name, "shinyapps", shinyapps_server.url, token=shinyapps_server.token, secret=shinyapps_server.secret ) if old_server: click.echo('Updated shinyapps.io credential "%s".' % name) @@ -335,7 +335,7 @@ def add(name, server, api_key, insecure, cacert, token, secret, verbose): server_store.set( name, - 'connect', + "connect", real_server.url, real_server.api_key, real_server.insecure, @@ -522,64 +522,6 @@ def deploy(): pass -def _validate_deploy_to_args(name, url, api_key, insecure, ca_cert, api_key_is_required=True) -> api.RemoteServer: - # TODO (mslynch): accept non-saved credentials here (maybe)? - """ - Validate that the user gave us enough information to talk to a Connect server. - - :param name: the nickname, if any, specified by the user. - :param url: the URL, if any, specified by the user. - :param api_key: the API key, if any, specified by the user. - :param insecure: a flag noting whether TLS host/validation should be skipped. - :param ca_cert: the name of a CA certs file containing certificates to use. - :param api_key_is_required: a flag that notes whether the API key is required or may - be omitted. - :return: a ConnectServer or ShinyappsServer object that carries all the right info. - """ - ca_data = ca_cert and text_type(ca_cert.read()) - - if name and url: - raise RSConnectException("You must specify only one of -n/--name or -s/--server, not both") - - server_data = server_store.resolve(name, url, api_key, insecure, ca_data) - - # This can happen if the user specifies neither --name or --server and there's not - # a single default to go with. - if not server_data.url: - raise api.RSConnectException("You must specify one of -n/--name or -s/--server.") - - if server_data.target == "connect": - - connect_server = api.RSConnectServer(server_data.url, None, server_data.insecure, server_data.ca_data) - - # If our info came from the command line, make sure the URL really works. - if not server_data.from_store: - connect_server, _ = test_server(connect_server) - - connect_server.api_key = server_data.api_key - - if not connect_server.api_key: - if api_key_is_required: - raise RSConnectException('An API key must be specified for "%s".' % connect_server.url) - return connect_server - - # If our info came from the command line, make sure the key really works. - if not server_data.from_store: - _ = test_api_key(connect_server) - - return connect_server - else: - # TODO (mslynch): replace nickname with account name - if not server_data.token: - raise api.RSConnectException("A token must be specified.") - if not server_data.token: - raise api.RSConnectException("A secret must be specified.") - - server = api.ShinyappsServer(server_data.url, server_data.name, server_data.token, server_data.secret) - test_shinyapps_server(server) - return server - - def _warn_on_ignored_manifest(directory): """ Checks for the existence of a file called manifest.json in the given directory. diff --git a/rsconnect/models.py b/rsconnect/models.py index 82fbac97..6c7a1d73 100644 --- a/rsconnect/models.py +++ b/rsconnect/models.py @@ -96,6 +96,18 @@ class AppModes(object): STATIC_QUARTO, ] + _cloud_to_connect_modes = { + "shiny": SHINY, + "rmarkdown_static": RMD, + "rmarkdown": SHINY_RMD, + "plumber": PLUMBER, + "flask": PYTHON_API, + "dash": DASH_APP, + "streamlit": STREAMLIT_APP, + "fastapi": PYTHON_FASTAPI, + "bokeh": BOKEH_APP, + } + @classmethod def get_by_ordinal(cls, ordinal, return_unknown=False): """Get an AppMode by its associated ordinal (integer)""" @@ -125,6 +137,10 @@ def get_by_extension(cls, extension, return_unknown=False): return_unknown, ) + @classmethod + def get_by_cloud_name(cls, name): + return cls._cloud_to_connect_modes.get(name, cls.UNKNOWN) + @classmethod def _find_by(cls, predicate, message, return_unknown): for mode in cls._modes: diff --git a/tests/test_main.py b/tests/test_main.py index 58406b5d..3f5f60eb 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -15,27 +15,11 @@ require_api_key, require_connect, ) -from rsconnect.exception import RSConnectException -from rsconnect.main import cli, _validate_deploy_to_args, server_store +from rsconnect.main import cli from rsconnect import VERSION class TestMain(TestCase): - def test_validate_deploy_to_args(self): - server_store.set("fake", "connect", "http://example.com", None) - - try: - with self.assertRaises(RSConnectException): - _validate_deploy_to_args("name", "url", None, False, None) - - with self.assertRaises(RSConnectException): - _validate_deploy_to_args(None, None, None, False, None) - - with self.assertRaises(RSConnectException): - _validate_deploy_to_args("fake", None, None, False, None) - finally: - server_store.remove_by_name("fake") - def require_connect(self): connect_server = os.environ.get("CONNECT_SERVER", None) if connect_server is None: From 8a063591211da8866104f9bf796f70f56bcf5985 Mon Sep 17 00:00:00 2001 From: Matthew Lynch Date: Wed, 29 Jun 2022 15:28:37 -0500 Subject: [PATCH 08/57] working manifest shinyapps.io deploys --- rsconnect/api.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rsconnect/api.py b/rsconnect/api.py index 76f11ca1..107f0889 100644 --- a/rsconnect/api.py +++ b/rsconnect/api.py @@ -511,7 +511,8 @@ def make_bundle(self, func: Callable, *args, **kwargs): d = self.state d["title_is_default"] = not bool(title) d["title"] = title or _default_title(path) - d["deployment_name"] = self.make_deployment_name(d["title"], app_id is None) + force_unique_name = app_id is None and isinstance(self.remote_server, RSConnectServer) + d["deployment_name"] = self.make_deployment_name(d["title"], force_unique_name) try: bundle = func(*args, **kwargs) From e3b67a39fae1b62800c9e040f8efb329099703c9 Mon Sep 17 00:00:00 2001 From: Matthew Lynch Date: Wed, 29 Jun 2022 17:52:42 -0500 Subject: [PATCH 09/57] add account option to `add` command --- rsconnect/api.py | 3 +-- rsconnect/main.py | 40 +++++++++++++++++++++++++++++++++------- rsconnect/metadata.py | 11 +++++++---- 3 files changed, 41 insertions(+), 13 deletions(-) diff --git a/rsconnect/api.py b/rsconnect/api.py index 107f0889..5120a3f9 100644 --- a/rsconnect/api.py +++ b/rsconnect/api.py @@ -397,7 +397,7 @@ def setup_remote_server( ) else: self.remote_server = ShinyappsServer( - server_data.url, server_data.name, server_data.token, server_data.secret + server_data.url, server_data.account, server_data.token, server_data.secret ) def setup_client(self, cookies=None, timeout=30, **kwargs): @@ -1103,7 +1103,6 @@ def do_bundle_deploy(remote_server: RemoteServer, app_id, name, title, title_is_ remote_server.handle_bad_response(result) return result else: - raise Exception('whuh oh') contents = bundle.read() bundle_size = len(contents) bundle_hash = hashlib.md5(contents).hexdigest() diff --git a/rsconnect/main.py b/rsconnect/main.py index e0961a3b..945cf456 100644 --- a/rsconnect/main.py +++ b/rsconnect/main.py @@ -285,6 +285,12 @@ def _test_shinyappsio_creds(server: api.ShinyappsServer): type=click.File(), help="The path to trusted TLS CA certificates.", ) +@click.option( + "--account", + "-a", + envvar="SHINYAPPS_ACCOUNT", + help="The shinyapps.io account name.", +) @click.option( "--token", "-T", @@ -298,11 +304,11 @@ def _test_shinyappsio_creds(server: api.ShinyappsServer): help="The shinyapps.io token secret.", ) @click.option("--verbose", "-v", is_flag=True, help="Print detailed messages.") -def add(name, server, api_key, insecure, cacert, token, secret, verbose): +def add(name, server, api_key, insecure, cacert, account, token, secret, verbose): set_verbosity(verbose) connect_options = {"--api-key": api_key, "--insecure": insecure, "--cacert": cacert} - shinyapps_options = {"--token": token, "--secret": secret} + shinyapps_options = {"--token": token, "--secret": secret, "--account": account} present_connect_options = [k for k, v in connect_options.items() if v] present_shinyapps_options = [k for k, v in shinyapps_options.items() if v] @@ -315,15 +321,21 @@ def add(name, server, api_key, insecure, cacert, token, secret, verbose): old_server = server_store.get_by_name(name) - if present_shinyapps_options: - if len(present_shinyapps_options) != 2: - raise api.RSConnectException("--token and --secret must both be provided for shinyapps.io.") + print(locals()) - shinyapps_server = api.ShinyappsServer(server or "https://api.shinyapps.io", name, token, secret) + if present_shinyapps_options: + if len(present_shinyapps_options) != 3: + raise api.RSConnectException("--account, --token, and --secret must all be provided for shinyapps.io.") + shinyapps_server = api.ShinyappsServer(server or "https://api.shinyapps.io", account, token, secret) _test_shinyappsio_creds(shinyapps_server) server_store.set( - name, "shinyapps", shinyapps_server.url, token=shinyapps_server.token, secret=shinyapps_server.secret + name, + "shinyapps", + shinyapps_server.url, + account=shinyapps_server.account_name, + token=shinyapps_server.token, + secret=shinyapps_server.secret, ) if old_server: click.echo('Updated shinyapps.io credential "%s".' % name) @@ -728,6 +740,18 @@ def deploy_notebook( ) @server_args @content_args +@click.option( + "--token", + "-T", + envvar="SHINYAPPS_TOKEN", + help="The shinyapps.io token.", +) +@click.option( + "--secret", + "-S", + envvar="SHINYAPPS_SECRET", + help="The shinyapps.io token secret.", +) @click.argument("file", type=click.Path(exists=True, dir_okay=True, file_okay=True)) @cli_exception_handler def deploy_manifest( @@ -736,6 +760,8 @@ def deploy_manifest( api_key: str, insecure: bool, cacert: typing.IO, + token: str, + secret: str, new: bool, app_id: str, title: str, diff --git a/rsconnect/metadata.py b/rsconnect/metadata.py index e9cad010..63bad2a0 100644 --- a/rsconnect/metadata.py +++ b/rsconnect/metadata.py @@ -228,6 +228,7 @@ def __init__( api_key: typing.Optional[str] = None, insecure: typing.Optional[bool] = None, ca_data: typing.Optional[str] = None, + account: typing.Optional[str] = None, token: typing.Optional[str] = None, secret: typing.Optional[str] = None, ): @@ -238,6 +239,7 @@ def __init__( self.api_key = api_key self.insecure = insecure self.ca_data = ca_data + self.account = account self.token = token self.secret = secret @@ -277,7 +279,7 @@ def get_all_servers(self): """ return self._get_sorted_values(lambda s: s["name"]) - def set(self, name, target, url, api_key=None, insecure=False, ca_data=None, token=None, secret=None): + def set(self, name, target, url, api_key=None, insecure=False, ca_data=None, account=None, token=None, secret=None): """ Add (or update) information about a Connect server @@ -287,18 +289,18 @@ def set(self, name, target, url, api_key=None, insecure=False, ca_data=None, tok :param api_key: the API key to use to authenticate with the Connect server. :param insecure: a flag to disable TLS verification. :param ca_data: client side certificate data to use for TLS. + :param account: shinyapps.io account name. :param token: shinyapps.io token. - :param token: shinyapps.io secret. + :param secret: shinyapps.io secret. """ common_data = dict( target=target, - name=name, url=url, ) if target == "connect": target_data = dict(api_key=api_key, insecure=insecure, ca_cert=ca_data) else: - target_data = dict(token=token, secret=secret) + target_data = dict(account=account, token=token, secret=secret) self._set(name, {**common_data, **target_data}) def remove_by_name(self, name): @@ -363,6 +365,7 @@ def resolve(self, name, url, api_key, insecure, ca_data, target="connect"): insecure=entry.get("insecure"), ca_data=entry.get("ca_cert"), api_key=entry.get("api_key"), + account=entry.get("account"), token=entry.get("token"), secret=entry.get("secret"), ) From f3aafa89230bd34337ecad8d22e41c8c5b456f52 Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Tue, 28 Jun 2022 16:18:23 -0400 Subject: [PATCH 10/57] update setup_connect_server & setup_client for ShinyappsClient --- rsconnect/api.py | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/rsconnect/api.py b/rsconnect/api.py index 31df6ebf..37e627cb 100644 --- a/rsconnect/api.py +++ b/rsconnect/api.py @@ -502,20 +502,36 @@ def setup_connect_server( insecure: bool = False, cacert: IO = None, ca_data: str = None, + account_name: str = None, + token: str = None, + secret: str = None, ): if name and url: raise RSConnectException("You must specify only one of -n/--name or -s/--server, not both.") if not name and not url: raise RSConnectException("You must specify one of -n/--name or -s/--server.") - + if api_key and (token or secret): + raise RSConnectException("You must specify either only 1) an api key, or 2) a token and a secret.") if cacert and not ca_data: ca_data = text_type(cacert.read()) - url, api_key, insecure, ca_data, _ = ServerStore().resolve(name, url, api_key, insecure, ca_data) - self.connect_server = RSConnectServer(url, api_key, insecure, ca_data) + server_data = ServerStore().resolve(name, url, api_key, insecure, ca_data) + if server_data.api_key: + self.connect_server = RSConnectServer( + server_data.url, server_data.api_key, server_data.insecure, server_data.ca_data + ) + elif server_data.token and server_data.secret: + self.connect_server = ShinyappsServer(server_data.url, account_name, server_data.token, server_data.secret) + else: + raise RSConnectException("Unable to infer Connect server.") def setup_client(self, cookies=None, timeout=30, **kwargs): - self.client = RSConnectClient(self.connect_server, cookies, timeout) + if isinstance(self.connect_server, RSConnectServer): + self.client = RSConnectClient(self.connect_server, cookies, timeout) + elif isinstance(self.connect_server, ShinyappsServer): + self.client = ShinyappsClient(self.connect_server, timeout) + else: + raise RSConnectException("Unable to infer Connect client.") @property def state(self): From f959dc77fc5834916515fab1dc0e71b3150067c4 Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Thu, 30 Jun 2022 12:05:32 -0400 Subject: [PATCH 11/57] Split validate_server for Connect and Shinyapps --- rsconnect/api.py | 73 +++++++++++++++++++++++++++++++++++------------- 1 file changed, 53 insertions(+), 20 deletions(-) diff --git a/rsconnect/api.py b/rsconnect/api.py index 37e627cb..cf0902a0 100644 --- a/rsconnect/api.py +++ b/rsconnect/api.py @@ -545,6 +545,27 @@ def pipe(self, func, *args, **kwargs): @cls_logged("Validating server...") def validate_server( + self, + name: str = None, + url: str = None, + api_key: str = None, + insecure: bool = False, + cacert: IO = None, + api_key_is_required: bool = False, + account_name: str = None, + token: str = None, + secret: str = None, + ): + if (url and api_key) or isinstance(self.connect_server, RSConnectServer): + self.validate_connect_server(name, url, api_key, insecure, cacert, api_key_is_required) + elif (url and token and secret) or isinstance(self.connect_server, ShinyappsServer): + self.validate_shinyapps_server(url, account_name, token, secret) + else: + raise RSConnectException("Unable to validate server from information provided.") + + return self + + def validate_connect_server( self, name: str = None, url: str = None, @@ -569,7 +590,6 @@ def validate_server( api_key = api_key or self.connect_server.api_key insecure = insecure or self.connect_server.insecure api_key_is_required = api_key_is_required or self.get("api_key_is_required", **kwargs) - server_store = ServerStore() if cacert: ca_data = text_type(cacert.read()) @@ -581,19 +601,11 @@ def validate_server( if not name and not url: raise RSConnectException("You must specify one of -n/--name or -s/--server.") - real_server, api_key, insecure, ca_data, from_store = server_store.resolve( - name, url, api_key, insecure, ca_data - ) - - # This can happen if the user specifies neither --name or --server and there's not - # a single default to go with. - if not real_server: - raise RSConnectException("You must specify one of -n/--name or -s/--server.") - - connect_server = RSConnectServer(real_server, None, insecure, ca_data) + server_data = ServerStore().resolve(name, url, api_key, insecure, ca_data) + connect_server = RSConnectServer(url, None, insecure, ca_data) # If our info came from the command line, make sure the URL really works. - if not from_store: + if not server_data.from_store: self.server_settings connect_server.api_key = api_key @@ -604,14 +616,30 @@ def validate_server( return self # If our info came from the command line, make sure the key really works. - if not from_store: - _ = self.verify_api_key() + if not server_data.from_store: + _ = self.verify_api_key(connect_server) self.connect_server = connect_server self.client = RSConnectClient(self.connect_server) return self + def validate_shinyapps_server( + self, url: str = None, account_name: str = None, token: str = None, secret: str = None, **kwargs + ): + url = url or self.connect_server.url + account_name = account_name or self.connect_server.account_name + token = token or self.connect_server.token + secret = secret or self.connect_server.secret + server = ShinyappsServer(url, account_name, token, secret) + + with ShinyappsClient(server) as client: + try: + result = client.get_current_user() + server.handle_bad_response(result) + except RSConnectException as exc: + raise RSConnectException(f"Failed to verify with shinyapps.io ({str(exc)}).") + @cls_logged("Making bundle ...") def make_bundle(self, func: Callable, *args, **kwargs): path = ( @@ -815,16 +843,21 @@ def server_settings(self): raise RSConnectException("There is an SSL/TLS configuration problem: %s" % ssl_error) return result - def verify_api_key(self): + def verify_api_key(self, server=None): """ Verify that an API Key may be used to authenticate with the given RStudio Connect server. If the API key verifies, we return the username of the associated user. """ - result = self.client.me() - if isinstance(result, HTTPResponse): - if result.json_data and "code" in result.json_data and result.json_data["code"] == 30: - raise RSConnectException("The specified API key is not valid.") - raise RSConnectException("Could not verify the API key: %s %s" % (result.status, result.reason)) + if not server: + server = self.connect_server + if isinstance(server, ShinyappsServer): + raise RSConnectException("Shinnyapps server does not use an API key.") + with RSConnectClient(server) as client: + result = client.me() + if isinstance(result, HTTPResponse): + if result.json_data and "code" in result.json_data and result.json_data["code"] == 30: + raise RSConnectException("The specified API key is not valid.") + raise RSConnectException("Could not verify the API key: %s %s" % (result.status, result.reason)) return self @property From cc22ea02a8aec8f3c70e49404b3ac8efb3ef6358 Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Thu, 30 Jun 2022 12:06:39 -0400 Subject: [PATCH 12/57] remove target in metadata to maintain backward compatibility --- rsconnect/metadata.py | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/rsconnect/metadata.py b/rsconnect/metadata.py index 4d838409..cc18d398 100644 --- a/rsconnect/metadata.py +++ b/rsconnect/metadata.py @@ -222,7 +222,6 @@ class ServerData: def __init__( self, name: str, - target: str, url: str, from_store: bool, api_key: typing.Optional[str] = None, @@ -232,7 +231,6 @@ def __init__( secret: typing.Optional[str] = None, ): self.name = name - self.target = target self.url = url self.from_store = from_store self.api_key = api_key @@ -277,7 +275,7 @@ def get_all_servers(self): """ return self._get_sorted_values(lambda s: s["name"]) - def set(self, name, target, url, api_key=None, insecure=False, ca_data=None, token=None, secret=None): + def set(self, name, url, api_key=None, insecure=False, ca_data=None, token=None, secret=None): """ Add (or update) information about a Connect server @@ -291,14 +289,10 @@ def set(self, name, target, url, api_key=None, insecure=False, ca_data=None, tok :param token: shinyapps.io secret. """ common_data = dict( - target=target, name=name, url=url, ) - if target == "connect": - target_data = dict(api_key=api_key, insecure=insecure, ca_cert=ca_data) - else: - target_data = dict(token=token, secret=secret) + target_data = dict(api_key=api_key, insecure=insecure, ca_data=ca_data, token=token, secret=secret) self._set(name, {**common_data, **target_data}) def remove_by_name(self, name): @@ -317,7 +311,7 @@ def remove_by_url(self, url): """ return self._remove_by_value_attr("name", "url", url) - def resolve(self, name, url, api_key, insecure, ca_data, target="connect"): + def resolve(self, name, url, api_key, insecure, ca_data): """ This function will resolve the given inputs into a set of server information. It assumes that either `name` or `url` is provided. @@ -357,7 +351,6 @@ def resolve(self, name, url, api_key, insecure, ca_data, target="connect"): if entry: return ServerData( name, - entry["target"], entry["url"], True, insecure=entry.get("insecure"), @@ -369,8 +362,6 @@ def resolve(self, name, url, api_key, insecure, ca_data, target="connect"): else: return ServerData( name, - # TODO (mslynch): this function needs to receive target - "connect", url, False, insecure=insecure, From 9f8fcffb93f7eb5d5033dd759394e96d8756d08e Mon Sep 17 00:00:00 2001 From: Matthew Lynch Date: Thu, 30 Jun 2022 11:34:06 -0500 Subject: [PATCH 13/57] working no-server-nickname deploy for shinyapps --- rsconnect/api.py | 110 +++++++++++++--------------------------- rsconnect/main.py | 12 ++++- rsconnect/metadata.py | 13 ++--- rsconnect/validation.py | 47 +++++++++++++++++ 4 files changed, 100 insertions(+), 82 deletions(-) create mode 100644 rsconnect/validation.py diff --git a/rsconnect/api.py b/rsconnect/api.py index 5120a3f9..6ba03df1 100644 --- a/rsconnect/api.py +++ b/rsconnect/api.py @@ -23,6 +23,8 @@ from warnings import warn from six import text_type import gc + +from . import validation from .bundle import fake_module_file_from_directory from .http_support import HTTPResponse, HTTPServer, append_to_path, CookieJar from .log import logger, connect_logger, cls_logged, console_logger @@ -337,6 +339,7 @@ def __init__( cacert: IO = None, ca_data: str = None, cookies=None, + account=None, token: str = None, secret: str = None, timeout: int = 30, @@ -345,7 +348,17 @@ def __init__( ) -> None: self.reset() self._d = kwargs - self.setup_remote_server(name, url or kwargs.get("server"), api_key, insecure, cacert, ca_data) + self.setup_remote_server( + name=name, + url=url or kwargs.get("server"), + api_key=api_key, + insecure=insecure, + cacert=cacert, + ca_data=ca_data, + account=account, + token=token, + secret=secret, + ) self.setup_client(cookies, timeout) self.logger = logger @@ -380,18 +393,28 @@ def setup_remote_server( insecure: bool = False, cacert: IO = None, ca_data: str = None, + account: str = None, + token: str = None, + secret: str = None, ): - if name and url: - raise RSConnectException("You must specify only one of -n/--name or -s/--server, not both.") - if not name and not url: - raise RSConnectException("You must specify one of -n/--name or -s/--server.") + target = validation.validate_connection_options( + name=name, + url=url, + api_key=api_key, + insecure=insecure, + cacert=cacert, + account=account, + token=token, + secret=secret, + ) if cacert and not ca_data: ca_data = text_type(cacert.read()) - server_data = ServerStore().resolve(name, url, api_key, insecure, ca_data) + server_data = ServerStore().resolve(name, url, api_key, insecure, ca_data, account, token, secret) + self.is_server_from_store = server_data.from_store - if server_data.api_key is not None: + if target == "connect": self.remote_server = RSConnectServer( server_data.url, server_data.api_key, server_data.insecure, server_data.ca_data ) @@ -417,78 +440,17 @@ def pipe(self, func, *args, **kwargs): return func(*args, **kwargs) @cls_logged("Validating server...") - def validate_server( - self, - name: str = None, - url: str = None, - api_key: str = None, - insecure: bool = False, - cacert: IO = None, - api_key_is_required: bool = False, - token: str = None, - secret: str = None, - **kwargs - ): + def validate_server(self): """ Validate that the user gave us enough information to talk to shinyapps.io or a Connect server. - - :param name: the nickname, if any, specified by the user. - :param url: the URL, if any, specified by the user. - :param api_key: the API key, if any, specified by the user. - :param insecure: a flag noting whether TLS host/validation should be skipped. - :param cacert: the file object of a CA certs file containing certificates to use. - :param api_key_is_required: a flag that notes whether the API key is required or may - be omitted. - :param token: The shinyapps.io authentication token. - :param secret: The shinyapps.io authentication secret. """ - url = url or self.remote_server.url - ca_data = None - if cacert: - ca_data = text_type(cacert.read()) if isinstance(self.remote_server, RSConnectServer): - api_key = api_key or self.remote_server.api_key - insecure = insecure or self.remote_server.insecure - if not ca_data: - ca_data = self.remote_server.ca_data + # If our info came from the command line, make sure the URL and key really work. + if not self.is_server_from_store: + self.server_settings() + _ = self.verify_api_key() else: - token = token or self.remote_server.token - secret = secret or self.remote_server.secret - - api_key_is_required = api_key_is_required or self.get("api_key_is_required", **kwargs) - server_store = ServerStore() - - if name and url: - raise RSConnectException("You must specify only one of -n/--name or -s/--server, not both") - if not name and not url: - raise RSConnectException("You must specify one of -n/--name or -s/--server.") - - server_data = server_store.resolve(name, url, api_key, insecure, ca_data) - - # This can happen if the user specifies neither --name or --server and there's not - # a single default to go with. - if not server_data.url: - raise RSConnectException("You must specify one of -n/--name or -s/--server.") - - connect_server = RSConnectServer(server_data.url, None, insecure, ca_data) - - # If our info came from the command line, make sure the URL really works. - if not server_data.from_store: - self.server_settings() - - connect_server.api_key = api_key - - if not connect_server.api_key: - if api_key_is_required: - raise RSConnectException('An API key must be specified for "%s".' % connect_server.url) - return self - - # If our info came from the command line, make sure the key really works. - if not server_data.from_store: - _ = self.verify_api_key() - - self.remote_server = connect_server - self.client = RSConnectClient(self.remote_server) + self.client.get_current_user() return self diff --git a/rsconnect/main.py b/rsconnect/main.py index 945cf456..7d5df3a0 100644 --- a/rsconnect/main.py +++ b/rsconnect/main.py @@ -305,6 +305,7 @@ def _test_shinyappsio_creds(server: api.ShinyappsServer): ) @click.option("--verbose", "-v", is_flag=True, help="Print detailed messages.") def add(name, server, api_key, insecure, cacert, account, token, secret, verbose): + set_verbosity(verbose) connect_options = {"--api-key": api_key, "--insecure": insecure, "--cacert": cacert} @@ -321,8 +322,6 @@ def add(name, server, api_key, insecure, cacert, account, token, secret, verbose old_server = server_store.get_by_name(name) - print(locals()) - if present_shinyapps_options: if len(present_shinyapps_options) != 3: raise api.RSConnectException("--account, --token, and --secret must all be provided for shinyapps.io.") @@ -740,6 +739,14 @@ def deploy_notebook( ) @server_args @content_args +@click.option( + "--account", + "-a", + envvar="SHINYAPPS_ACCOUNT", + help="The shinyapps.io account name.", +) +@server_args +@content_args @click.option( "--token", "-T", @@ -760,6 +767,7 @@ def deploy_manifest( api_key: str, insecure: bool, cacert: typing.IO, + account: str, token: str, secret: str, new: bool, diff --git a/rsconnect/metadata.py b/rsconnect/metadata.py index 63bad2a0..6838e79e 100644 --- a/rsconnect/metadata.py +++ b/rsconnect/metadata.py @@ -222,7 +222,6 @@ class ServerData: def __init__( self, name: str, - target: str, url: str, from_store: bool, api_key: typing.Optional[str] = None, @@ -233,7 +232,6 @@ def __init__( secret: typing.Optional[str] = None, ): self.name = name - self.target = target self.url = url self.from_store = from_store self.api_key = api_key @@ -319,7 +317,7 @@ def remove_by_url(self, url): """ return self._remove_by_value_attr("name", "url", url) - def resolve(self, name, url, api_key, insecure, ca_data, target="connect"): + def resolve(self, name, url, api_key, insecure, ca_data, account, token, secret): """ This function will resolve the given inputs into a set of server information. It assumes that either `name` or `url` is provided. @@ -340,6 +338,9 @@ def resolve(self, name, url, api_key, insecure, ca_data, target="connect"): :param api_key: the API key provided on the command line. :param insecure: the insecure flag provided on the command line. :param ca_data: the CA certification data provided on the command line. + :param account: the shinyapps.io account name. + :param :token: the shinyapps.io auth token. + :param :secret: the shinyapps.io auth secret. :return: the information needed to interact with the resolved server and whether it came from the store or the arguments. """ @@ -359,7 +360,6 @@ def resolve(self, name, url, api_key, insecure, ca_data, target="connect"): if entry: return ServerData( name, - entry["target"], entry["url"], True, insecure=entry.get("insecure"), @@ -372,13 +372,14 @@ def resolve(self, name, url, api_key, insecure, ca_data, target="connect"): else: return ServerData( name, - # TODO (mslynch): this function needs to receive target - "connect", url, False, insecure=insecure, ca_data=ca_data, api_key=api_key, + account=account, + token=token, + secret=secret, ) diff --git a/rsconnect/validation.py b/rsconnect/validation.py new file mode 100644 index 00000000..0018a01d --- /dev/null +++ b/rsconnect/validation.py @@ -0,0 +1,47 @@ +import typing + +from . import api + + +def _get_present_options(options: dict[str, typing.Optional[str]]) -> list[str]: + return [k for k, v in options.items() if v] + + +def validate_connection_options(name, url, api_key, insecure, cacert, account, token, secret): + """ + Validates provided Connect or shinyapps.io connection options and returns which target to use given the provided + options. + """ + connect_options = {"-k/--api-key": api_key, "-i/--insecure": insecure, "-c/--cacert": cacert} + shinyapps_options = {"-T/--token": token, "-S/--secret": secret, "-a/--account": account} + options_mutually_exclusive_with_name = {"--server": url, **connect_options, **shinyapps_options} + + present_options_mutually_exclusive_with_name = _get_present_options(options_mutually_exclusive_with_name) + + if name and present_options_mutually_exclusive_with_name: + raise api.RSConnectException( + f"-n/--name cannot be specified in conjunction with options {', '.join(present_options_mutually_exclusive_with_name)}" + ) + if not name and not url: + raise api.RSConnectException( + "You must specify one of -n/--name OR -s/--server OR -a/--account, -T/--token, -S/--secret." + ) + + present_connect_options = _get_present_options(connect_options) + present_shinyapps_options = _get_present_options(shinyapps_options) + + if present_connect_options and present_shinyapps_options: + raise api.RSConnectException( + f"Connect options ({', '.join(present_connect_options)}) may not be passed alongside " + f"shinyapps.io options ({', '.join(present_shinyapps_options)})." + ) + + if present_shinyapps_options: + if len(present_shinyapps_options) != 3: + raise api.RSConnectException( + "-a/--account, -T/--token, and -S/--secret must all be provided for shinyapps.io." + ) + + return "shinyapps" + + return "connect" From b69d5efaed7b85e01ec76c5a38bae5d09fc02d77 Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Thu, 30 Jun 2022 13:02:51 -0400 Subject: [PATCH 14/57] update setup when token and secret not in ServerStore --- rsconnect/api.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/rsconnect/api.py b/rsconnect/api.py index 818ea86f..0fe7a473 100644 --- a/rsconnect/api.py +++ b/rsconnect/api.py @@ -397,6 +397,8 @@ def setup_connect_server( ) elif server_data.token and server_data.secret: self.connect_server = ShinyappsServer(server_data.url, account_name, server_data.token, server_data.secret) + elif token and secret: + self.connect_server = ShinyappsServer(url, account_name, token, secret) else: raise RSConnectException("Unable to infer Connect server.") From 7255c2bdd910b62584ce708cf1442cda26d162bc Mon Sep 17 00:00:00 2001 From: Matthew Lynch Date: Thu, 30 Jun 2022 12:03:53 -0500 Subject: [PATCH 15/57] python 3.5 compatibility --- rsconnect/actions.py | 2 +- rsconnect/api.py | 24 +++++++++++------------- rsconnect/main.py | 5 +++-- rsconnect/validation.py | 9 ++++++--- 4 files changed, 21 insertions(+), 19 deletions(-) diff --git a/rsconnect/actions.py b/rsconnect/actions.py index bd952c15..a90fa864 100644 --- a/rsconnect/actions.py +++ b/rsconnect/actions.py @@ -224,7 +224,7 @@ def test_shinyapps_server(server: api.ShinyappsServer): result = client.get_current_user() server.handle_bad_response(result) except api.RSConnectException as exc: - raise api.RSConnectException(f"Failed to verify with shinyapps.io ({str(exc)}).") + raise api.RSConnectException("Failed to verify with shinyapps.io ({}).".format(exc)) def test_api_key(connect_server): diff --git a/rsconnect/api.py b/rsconnect/api.py index 6ba03df1..725cfde0 100644 --- a/rsconnect/api.py +++ b/rsconnect/api.py @@ -1,9 +1,7 @@ """ RStudio Connect API client and utility functions """ -import abc import base64 -import calendar import datetime import hashlib import hmac @@ -17,8 +15,6 @@ from urllib import parse from urllib.parse import urlparse -import click - import re from warnings import warn from six import text_type @@ -563,9 +559,11 @@ def deploy_bundle( upload_url = prepare_deploy_result.presigned_url parsed_upload_url = urlparse(upload_url) - with S3Client(f"{parsed_upload_url.scheme}://{parsed_upload_url.netloc}", timeout=120) as s3_client: + with S3Client( + "{}://{}".format(parsed_upload_url.scheme, parsed_upload_url.netloc), timeout=120 + ) as s3_client: upload_result = s3_client.upload( - f"{parsed_upload_url.path}?{parsed_upload_url.query}", + "{}?{}".format(parsed_upload_url.path, parsed_upload_url.query), prepare_deploy_result.presigned_checksum, bundle_size, contents, @@ -862,7 +860,7 @@ def get_extra_headers(self, url, method, body): } def get_application(self, application_id): - return self.get(f"/v1/applications/{application_id}") + return self.get("/v1/applications/{}".format(application_id)) def create_application(self, account_id, application_name): application_data = { @@ -885,13 +883,13 @@ def create_bundle(self, application_id: int, content_type: str, content_length: return self.post("/v1/bundles", body=bundle_data) def set_bundle_status(self, bundle_id, bundle_status): - return self.post(f"/v1/bundles/{bundle_id}/status", body={"status": bundle_status}) + return self.post("/v1/bundles/{}/status".format(bundle_id), body={"status": bundle_status}) def deploy_application(self, bundle_id, app_id): - return self.post(f"/v1/applications/{app_id}/deploy", body={"bundle": bundle_id, "rebuild": False}) + return self.post("/v1/applications/{}/deploy".format(app_id), body={"bundle": bundle_id, "rebuild": False}) def get_task(self, task_id): - return self.get(f"/v1/tasks/{task_id}", query_params={"legacy": "true"}) + return self.get("/v1/tasks/{}".format(task_id), query_params={"legacy": "true"}) def get_current_user(self): return self.get("/v1/users/me") @@ -906,14 +904,14 @@ def wait_until_task_is_successful(self, task_id, timeout=60): status = task.json_data["status"] description = task.json_data["description"] - click.secho(f"Waiting: {status} - {description}") + print("\nWaiting: {} - {}".format(status, description)) if status == "success": break time.sleep(2) counter += 1 - click.secho(f"Task done: {description}") + print("Task done: {}".format(description)) def prepare_deploy(self, app_id: typing.Optional[str], app_name: str, bundle_size: int, bundle_hash: str): accounts = self.get_accounts() @@ -1074,7 +1072,7 @@ def do_bundle_deploy(remote_server: RemoteServer, app_id, name, title, title_is_ upload_url = prepare_deploy_result.presigned_url parsed_upload_url = urlparse(upload_url) - with S3Client(f"{parsed_upload_url.scheme}://{parsed_upload_url.netloc}", timeout=120) as client: + with S3Client("{}://{}".format(parsed_upload_url.scheme, parsed_upload_url.netloc), timeout=120) as client: upload_result = client.upload( upload_url, prepare_deploy_result.presigned_checksum, diff --git a/rsconnect/main.py b/rsconnect/main.py index 7d5df3a0..0a6d082e 100644 --- a/rsconnect/main.py +++ b/rsconnect/main.py @@ -316,8 +316,9 @@ def add(name, server, api_key, insecure, cacert, account, token, secret, verbose if present_connect_options and present_shinyapps_options: raise api.RSConnectException( - f"Connect options ({', '.join(present_connect_options)}) may not be passed alongside " - f"shinyapps.io options ({', '.join(present_shinyapps_options)})." + "Connect options ({}) may not be passed alongside shinyapps.io options ({}).".format( + ", ".join(present_connect_options), ", ".join(present_shinyapps_options) + ) ) old_server = server_store.get_by_name(name) diff --git a/rsconnect/validation.py b/rsconnect/validation.py index 0018a01d..609f3993 100644 --- a/rsconnect/validation.py +++ b/rsconnect/validation.py @@ -20,7 +20,9 @@ def validate_connection_options(name, url, api_key, insecure, cacert, account, t if name and present_options_mutually_exclusive_with_name: raise api.RSConnectException( - f"-n/--name cannot be specified in conjunction with options {', '.join(present_options_mutually_exclusive_with_name)}" + "-n/--name cannot be specified in conjunction with options {}".format( + ", ".join(present_options_mutually_exclusive_with_name) + ) ) if not name and not url: raise api.RSConnectException( @@ -32,8 +34,9 @@ def validate_connection_options(name, url, api_key, insecure, cacert, account, t if present_connect_options and present_shinyapps_options: raise api.RSConnectException( - f"Connect options ({', '.join(present_connect_options)}) may not be passed alongside " - f"shinyapps.io options ({', '.join(present_shinyapps_options)})." + "Connect options ({}) may not be passed alongside shinyapps.io options ({}).".format( + ", ".join(present_connect_options), ", ".join(present_shinyapps_options) + ) ) if present_shinyapps_options: From e4fea5884bb7f7105fb3b2efa0534e17a8b478ab Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Thu, 30 Jun 2022 13:22:32 -0400 Subject: [PATCH 16/57] drop parameters from ServerStore.resolve() --- rsconnect/api.py | 26 +++++++++++++++----------- rsconnect/metadata.py | 8 +------- tests/test_metadata.py | 6 +++--- 3 files changed, 19 insertions(+), 21 deletions(-) diff --git a/rsconnect/api.py b/rsconnect/api.py index 0fe7a473..8a786156 100644 --- a/rsconnect/api.py +++ b/rsconnect/api.py @@ -342,7 +342,7 @@ def __init__( ) -> None: self.reset() self._d = kwargs - self.setup_connect_server(name, url or kwargs.get("server"), api_key, insecure, cacert, ca_data) + self.setup_connect_server(name, url or kwargs.get("server"), api_key, insecure, cacert, ca_data, token, secret) self.setup_client(cookies, timeout) self.logger = logger @@ -390,17 +390,21 @@ def setup_connect_server( if cacert and not ca_data: ca_data = text_type(cacert.read()) - server_data = ServerStore().resolve(name, url, api_key, insecure, ca_data) - if server_data.api_key: - self.connect_server = RSConnectServer( - server_data.url, server_data.api_key, server_data.insecure, server_data.ca_data - ) - elif server_data.token and server_data.secret: - self.connect_server = ShinyappsServer(server_data.url, account_name, server_data.token, server_data.secret) + server_data = ServerStore().resolve(name, url) + if server_data.from_store: + url = server_data.url + api_key = server_data.api_key + insecure = server_data.insecure + ca_data = server_data.ca_data + token = server_data.token + secret = server_data.secret + + if api_key: + self.connect_server = RSConnectServer(url, api_key, insecure, ca_data) elif token and secret: - self.connect_server = ShinyappsServer(url, account_name, token, secret) + self.connect_server = ShinyappsServer(url, account_name, token, secret) else: - raise RSConnectException("Unable to infer Connect server.") + raise RSConnectException("Unable to infer Connect server type and setup server.") def setup_client(self, cookies=None, timeout=30, **kwargs): if isinstance(self.connect_server, RSConnectServer): @@ -489,7 +493,7 @@ def validate_connect_server( if not name and not url: raise RSConnectException("You must specify one of -n/--name or -s/--server.") - server_data = ServerStore().resolve(name, url, api_key, insecure, ca_data) + server_data = ServerStore().resolve(name, url) connect_server = RSConnectServer(url, None, insecure, ca_data) # If our info came from the command line, make sure the URL really works. diff --git a/rsconnect/metadata.py b/rsconnect/metadata.py index ee789fa9..e0a3365b 100644 --- a/rsconnect/metadata.py +++ b/rsconnect/metadata.py @@ -314,7 +314,7 @@ def remove_by_url(self, url): """ return self._remove_by_value_attr("name", "url", url) - def resolve(self, name, url, api_key, insecure, ca_data): + def resolve(self, name, url): """ This function will resolve the given inputs into a set of server information. It assumes that either `name` or `url` is provided. @@ -332,9 +332,6 @@ def resolve(self, name, url, api_key, insecure, ca_data): :param name: the nickname to look for. :param url: the Connect server URL to look for. - :param api_key: the API key provided on the command line. - :param insecure: the insecure flag provided on the command line. - :param ca_data: the CA certification data provided on the command line. :return: the information needed to interact with the resolved server and whether it came from the store or the arguments. """ @@ -368,9 +365,6 @@ def resolve(self, name, url, api_key, insecure, ca_data): name, url, False, - insecure=insecure, - ca_data=ca_data, - api_key=api_key, ) diff --git a/tests/test_metadata.py b/tests/test_metadata.py index 10c1bc3d..dc63c59c 100644 --- a/tests/test_metadata.py +++ b/tests/test_metadata.py @@ -93,7 +93,7 @@ def test_list(self): self.assertEqual(servers[2]["url"], "http://connect.local") def check_resolve_call(self, name, server, api_key, insecure, ca_cert, should_be_from_store): - server_data = self.server_store.resolve(name, server, api_key, insecure, ca_cert) + server_data = self.server_store.resolve(name, server) self.assertEqual(server_data.url, "http://connect.local") self.assertEqual(server_data.api_key, "notReallyAnApiKey") @@ -110,7 +110,7 @@ def test_resolve_by_url(self): def test_resolve_by_default(self): # with multiple entries, server None will not resolve by default name, server, api_key, insecure, ca_cert = None, None, None, None, None - server_data = self.server_store.resolve(name, server, api_key, insecure, ca_cert) + server_data = self.server_store.resolve(name, server) self.assertEqual(server_data.url, None) # with only a single entry, server None will resolve to that entry @@ -126,7 +126,7 @@ def test_resolve_from_args(self): True, "fake-cert", ) - server_data = self.server_store.resolve(name, server, api_key, insecure, ca_cert) + server_data = self.server_store.resolve(name, server) self.assertEqual(server_data.url, "https://secured.connect") self.assertEqual(server_data.api_key, "an-api-key") From cea8e6039ca350f48b497104920989c80ddc48a0 Mon Sep 17 00:00:00 2001 From: Matthew Lynch Date: Thu, 30 Jun 2022 13:11:20 -0500 Subject: [PATCH 17/57] reuse RSConnectExecutor deploy_bundle in actions.deploy_bundle --- rsconnect/actions.py | 12 +++++++++- rsconnect/api.py | 53 +------------------------------------------- 2 files changed, 12 insertions(+), 53 deletions(-) diff --git a/rsconnect/actions.py b/rsconnect/actions.py index a90fa864..0926a6a1 100644 --- a/rsconnect/actions.py +++ b/rsconnect/actions.py @@ -1736,7 +1736,17 @@ def deploy_bundle( :return: application information about the deploy. This includes the ID of the task that may be queried for deployment progress. """ - return api.do_bundle_deploy(remote_server, app_id, name, title, title_is_default, bundle, env_vars) + ce = RSConnectExecutor( + server=remote_server, + app_id=app_id, + name=name, + title=title, + title_is_default=title_is_default, + bundle=bundle, + env_vars=env_vars, + ) + ce.deploy_bundle() + return ce.state["deployed_info"] def spool_deployment_log(connect_server, app, log_callback): diff --git a/rsconnect/api.py b/rsconnect/api.py index 47ea9e6e..9ce8dc9d 100644 --- a/rsconnect/api.py +++ b/rsconnect/api.py @@ -424,7 +424,7 @@ def setup_remote_server( secret = server_data.secret self.is_server_from_store = server_data.from_store - if target == 'connect': + if target == "connect": self.remote_server = RSConnectServer(url, api_key, insecure, ca_data) else: self.remote_server = ShinyappsServer(url, account, token, secret) @@ -1074,57 +1074,6 @@ def get_app_config(connect_server, app_id): return result -def do_bundle_deploy(connect_server: RemoteServer, app_id, name, title, title_is_default, bundle, env_vars): - """ - Deploys the specified bundle. - - :param connect_server: the server information. - :param app_id: the ID of the app to deploy, if this is a redeploy. - :param name: the name for the deploy. - :param title: the title for the deploy. - :param title_is_default: a flag noting whether the title carries a defaulted value. - :param bundle: the bundle to deploy. - :param env_vars: list of NAME=VALUE pairs for the app environment - :return: application information about the deploy. This includes the ID of the - task that may be queried for deployment progress. - """ - if isinstance(connect_server, RSConnectServer): - with RSConnectClient(connect_server, timeout=120) as client: - result = client.deploy(app_id, name, title, title_is_default, bundle, env_vars) - connect_server.handle_bad_response(result) - return result - else: - contents = bundle.read() - bundle_size = len(contents) - bundle_hash = hashlib.md5(contents).hexdigest() - - with ShinyappsClient(connect_server, timeout=120) as client: - prepare_deploy_result = client.prepare_deploy(app_id, name, bundle_size, bundle_hash) - - upload_url = prepare_deploy_result.presigned_url - parsed_upload_url = urlparse(upload_url) - with S3Client("{}://{}".format(parsed_upload_url.scheme, parsed_upload_url.netloc), timeout=120) as client: - upload_result = client.upload( - upload_url, - prepare_deploy_result.presigned_checksum, - bundle_size, - contents, - ) - S3Server(upload_url).handle_bad_response(upload_result) - - with ShinyappsClient(connect_server, timeout=120) as client: - client.do_deploy(prepare_deploy_result.bundle_id, prepare_deploy_result.app_id) - - webbrowser.open_new(prepare_deploy_result.app_url) - - return { - "app_url": prepare_deploy_result.app_url, - "app_id": prepare_deploy_result.app_id, - "app_guid": None, - "title": title, - } - - def emit_task_log( connect_server, app_id, From 86f9b5f32b1c8a67994ca9a8a4320ec6deb417c2 Mon Sep 17 00:00:00 2001 From: Matthew Lynch Date: Thu, 30 Jun 2022 13:56:32 -0500 Subject: [PATCH 18/57] fix tests --- rsconnect/api.py | 2 +- rsconnect/metadata.py | 9 ++++++--- tests/test_main.py | 4 +++- tests/test_metadata.py | 30 ++++++++++++++++-------------- 4 files changed, 26 insertions(+), 19 deletions(-) diff --git a/rsconnect/api.py b/rsconnect/api.py index 9ce8dc9d..9c43acad 100644 --- a/rsconnect/api.py +++ b/rsconnect/api.py @@ -455,7 +455,7 @@ def validate_server(self): if isinstance(self.remote_server, RSConnectServer): # If our info came from the command line, make sure the URL and key really work. if not self.is_server_from_store: - self.server_settings() + self.server_settings _ = self.verify_api_key() else: self.client.get_current_user() diff --git a/rsconnect/metadata.py b/rsconnect/metadata.py index e0a3365b..fe5e999d 100644 --- a/rsconnect/metadata.py +++ b/rsconnect/metadata.py @@ -277,7 +277,7 @@ def get_all_servers(self): """ return self._get_sorted_values(lambda s: s["name"]) - def set(self, name, url, api_key=None, insecure=False, ca_data=None, token=None, secret=None): + def set(self, name, url, target, api_key=None, insecure=False, ca_data=None, account=None, token=None, secret=None): """ Add (or update) information about a Connect server @@ -295,7 +295,10 @@ def set(self, name, url, api_key=None, insecure=False, ca_data=None, token=None, name=name, url=url, ) - target_data = dict(api_key=api_key, insecure=insecure, ca_data=ca_data, token=token, secret=secret) + if target == "connect": + target_data = dict(api_key=api_key, insecure=insecure, ca_cert=ca_data) + else: + target_data = dict(account=account, token=token, secret=secret) self._set(name, {**common_data, **target_data}) def remove_by_name(self, name): @@ -354,7 +357,7 @@ def resolve(self, name, url): entry["url"], True, insecure=entry.get("insecure"), - ca_data=entry.get("ca_data"), + ca_data=entry.get("ca_cert"), api_key=entry.get("api_key"), account=entry.get("account"), token=entry.get("token"), diff --git a/tests/test_main.py b/tests/test_main.py index 3f5f60eb..18ee081d 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -140,6 +140,8 @@ def test_add_shinyapps_missing_options(self): ], ) self.assertEqual(result.exit_code, 1, result.output) - self.assertEqual(str(result.exception), "--token and --secret must both be provided for shinyapps.io.") + self.assertEqual( + str(result.exception), "--account, --token, and --secret must all be provided for shinyapps.io." + ) finally: os.environ["CONNECT_API_KEY"] = original_api_key_value diff --git a/tests/test_metadata.py b/tests/test_metadata.py index dc63c59c..7892f393 100644 --- a/tests/test_metadata.py +++ b/tests/test_metadata.py @@ -17,9 +17,16 @@ def setUp(self): self.server_store_path = join(self.tempDir, "servers.json") self.assertFalse(exists(self.server_store_path)) - self.server_store.set("foo", "connect", "http://connect.local", "notReallyAnApiKey", ca_data="/certs/connect") - self.server_store.set("bar", "connect", "http://connect.remote", "differentApiKey", insecure=True) - self.server_store.set("baz", "shinyapps", "https://shinyapps.io", token="someToken", secret="c29tZVNlY3JldAo=") + self.server_store.set("foo", "http://connect.local", "connect", "notReallyAnApiKey", ca_data="/certs/connect") + self.server_store.set("bar", "http://connect.remote", "connect", "differentApiKey", insecure=True) + self.server_store.set( + "baz", + "https://shinyapps.io", + "shinyapps", + account="someAccount", + token="someToken", + secret="c29tZVNlY3JldAo=", + ) self.assertEqual(len(self.server_store.get_all_servers()), 3, "Unexpected servers after setup") def tearDown(self): @@ -30,7 +37,6 @@ def test_add(self): self.assertEqual( self.server_store.get_by_name("foo"), dict( - target="connect", name="foo", url="http://connect.local", api_key="notReallyAnApiKey", @@ -42,7 +48,6 @@ def test_add(self): self.assertEqual( self.server_store.get_by_name("bar"), dict( - target="connect", name="bar", url="http://connect.remote", api_key="differentApiKey", @@ -54,9 +59,9 @@ def test_add(self): self.assertEqual( self.server_store.get_by_name("baz"), dict( - target="shinyapps", name="baz", url="https://shinyapps.io", + account="someAccount", token="someToken", secret="c29tZVNlY3JldAo=", ), @@ -119,19 +124,16 @@ def test_resolve_by_default(self): self.check_resolve_call(None, None, None, None, None, True) def test_resolve_from_args(self): - name, server, api_key, insecure, ca_cert = ( + name, server = ( None, "https://secured.connect", - "an-api-key", - True, - "fake-cert", ) server_data = self.server_store.resolve(name, server) self.assertEqual(server_data.url, "https://secured.connect") - self.assertEqual(server_data.api_key, "an-api-key") - self.assertTrue(server_data.insecure) - self.assertEqual(server_data.ca_data, "fake-cert") + self.assertEqual(server_data.api_key, None) + self.assertEqual(server_data.insecure, None) + self.assertEqual(server_data.ca_data, None) self.assertFalse(server_data.from_store) def test_save_and_load(self): @@ -141,7 +143,7 @@ def test_save_and_load(self): self.assertFalse(exists(path)) - server_store.set("foo", "connect", "http://connect.local", "notReallyAnApiKey", ca_data="/certs/connect") + server_store.set("foo", "http://connect.local", "connect", "notReallyAnApiKey", ca_data="/certs/connect") self.assertTrue(exists(path)) From 69d15ebc51b23761c345500f978d059a38300c73 Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Thu, 30 Jun 2022 15:10:47 -0400 Subject: [PATCH 19/57] rename connect_server to remote_server --- rsconnect/api.py | 86 ++++++++++++++++++++++++------------------------ 1 file changed, 43 insertions(+), 43 deletions(-) diff --git a/rsconnect/api.py b/rsconnect/api.py index 8a786156..ea052f7f 100644 --- a/rsconnect/api.py +++ b/rsconnect/api.py @@ -342,7 +342,7 @@ def __init__( ) -> None: self.reset() self._d = kwargs - self.setup_connect_server(name, url or kwargs.get("server"), api_key, insecure, cacert, ca_data, token, secret) + self.setup_remote_server(name, url or kwargs.get("server"), api_key, insecure, cacert, ca_data, token, secret) self.setup_client(cookies, timeout) self.logger = logger @@ -358,7 +358,7 @@ def fromConnectServer(cls, connect_server, **kwargs): def reset(self): self._d = None - self.connect_server = None + self.remote_server = None self.client = None self.logger = None gc.collect() @@ -369,7 +369,7 @@ def drop_context(self): gc.collect() return self - def setup_connect_server( + def setup_remote_server( self, name: str = None, url: str = None, @@ -400,17 +400,17 @@ def setup_connect_server( secret = server_data.secret if api_key: - self.connect_server = RSConnectServer(url, api_key, insecure, ca_data) + self.remote_server = RSConnectServer(url, api_key, insecure, ca_data) elif token and secret: - self.connect_server = ShinyappsServer(url, account_name, token, secret) + self.remote_server = ShinyappsServer(url, account_name, token, secret) else: raise RSConnectException("Unable to infer Connect server type and setup server.") def setup_client(self, cookies=None, timeout=30, **kwargs): - if isinstance(self.connect_server, RSConnectServer): - self.client = RSConnectClient(self.connect_server, cookies, timeout) - elif isinstance(self.connect_server, ShinyappsServer): - self.client = ShinyappsClient(self.connect_server, timeout) + if isinstance(self.remote_server, RSConnectServer): + self.client = RSConnectClient(self.remote_server, cookies, timeout) + elif isinstance(self.remote_server, ShinyappsServer): + self.client = ShinyappsClient(self.remote_server, timeout) else: raise RSConnectException("Unable to infer Connect client.") @@ -437,9 +437,9 @@ def validate_server( token: str = None, secret: str = None, ): - if (url and api_key) or isinstance(self.connect_server, RSConnectServer): + if (url and api_key) or isinstance(self.remote_server, RSConnectServer): self.validate_connect_server(name, url, api_key, insecure, cacert, api_key_is_required) - elif (url and token and secret) or isinstance(self.connect_server, ShinyappsServer): + elif (url and token and secret) or isinstance(self.remote_server, ShinyappsServer): self.validate_shinyapps_server(url, account_name, token, secret) else: raise RSConnectException("Unable to validate server from information provided.") @@ -469,22 +469,22 @@ def validate_connect_server( :param token: The shinyapps.io authentication token. :param secret: The shinyapps.io authentication secret. """ - url = url or self.connect_server.url - api_key = api_key or self.connect_server.api_key - insecure = insecure or self.connect_server.insecure + url = url or self.remote_server.url + api_key = api_key or self.remote_server.api_key + insecure = insecure or self.remote_server.insecure api_key_is_required = api_key_is_required or self.get("api_key_is_required", **kwargs) ca_data = None if cacert: ca_data = text_type(cacert.read()) - if isinstance(self.connect_server, RSConnectServer): - api_key = api_key or self.connect_server.api_key - insecure = insecure or self.connect_server.insecure + if isinstance(self.remote_server, RSConnectServer): + api_key = api_key or self.remote_server.api_key + insecure = insecure or self.remote_server.insecure if not ca_data: - ca_data = self.connect_server.ca_data + ca_data = self.remote_server.ca_data else: - token = token or self.connect_server.token - secret = secret or self.connect_server.secret + token = token or self.remote_server.token + secret = secret or self.remote_server.secret api_key_is_required = api_key_is_required or self.get("api_key_is_required", **kwargs) @@ -511,18 +511,18 @@ def validate_connect_server( if not server_data.from_store: _ = self.verify_api_key(connect_server) - self.connect_server = connect_server - self.client = RSConnectClient(self.connect_server) + self.remote_server = connect_server + self.client = RSConnectClient(self.remote_server) return self def validate_shinyapps_server( self, url: str = None, account_name: str = None, token: str = None, secret: str = None, **kwargs ): - url = url or self.connect_server.url - account_name = account_name or self.connect_server.account_name - token = token or self.connect_server.token - secret = secret or self.connect_server.secret + url = url or self.remote_server.url + account_name = account_name or self.remote_server.account_name + token = token or self.remote_server.token + secret = secret or self.remote_server.secret server = ShinyappsServer(url, account_name, token, secret) with ShinyappsClient(server) as client: @@ -551,7 +551,7 @@ def make_bundle(self, func: Callable, *args, **kwargs): d = self.state d["title_is_default"] = not bool(title) d["title"] = title or _default_title(path) - force_unique_name = app_id is None and isinstance(self.connect_server, RSConnectServer) + force_unique_name = app_id is None and isinstance(self.remote_server, RSConnectServer) d["deployment_name"] = self.make_deployment_name(d["title"], force_unique_name) try: @@ -583,7 +583,7 @@ def check_server_capabilities(self, capability_functions): by default. """ # TODO (mslynch): check shinyapps capabilities? - if isinstance(self.connect_server, ShinyappsServer): + if isinstance(self.remote_server, ShinyappsServer): return self details = self.server_details @@ -615,7 +615,7 @@ def deploy_bundle( bundle = bundle or self.get("bundle") env_vars = env_vars or self.get("env_vars") - if isinstance(self.connect_server, RSConnectServer): + if isinstance(self.remote_server, RSConnectServer): result = self.client.deploy( app_id, deployment_name, @@ -624,7 +624,7 @@ def deploy_bundle( bundle, env_vars, ) - self.connect_server.handle_bad_response(result) + self.remote_server.handle_bad_response(result) self.state["deployed_info"] = result return self else: @@ -686,15 +686,15 @@ def emit_task_log( :param raise_on_error: whether to raise an exception when a task is failed, otherwise we return the task_result so we can record the exit code. """ - if isinstance(self.connect_server, RSConnectServer): + if isinstance(self.remote_server, RSConnectServer): app_id = app_id or self.state["deployed_info"]["app_id"] task_id = task_id or self.state["deployed_info"]["task_id"] log_lines, _ = self.client.wait_for_task( task_id, log_callback.info, abort_func, timeout, poll_wait, raise_on_error ) - self.connect_server.handle_bad_response(log_lines) + self.remote_server.handle_bad_response(log_lines) app_config = self.client.app_config(app_id) - self.connect_server.handle_bad_response(app_config) + self.remote_server.handle_bad_response(app_config) app_dashboard_url = app_config.get("config_url") log_callback.info("Deployment completed successfully.") log_callback.info("\t Dashboard content URL: %s", app_dashboard_url) @@ -715,7 +715,7 @@ def save_deployed_info(self, *args, **kwargs): deployed_info = self.get("deployed_info", *args, **kwargs) app_store.set( - self.connect_server.url, + self.remote_server.url, abspath(path), deployed_info["app_url"], deployed_info["app_id"], @@ -751,16 +751,16 @@ def validate_app_mode(self, *args, **kwargs): if app_id is None: # Possible redeployment - check for saved metadata. # Use the saved app information unless overridden by the user. - app_id, existing_app_mode = app_store.resolve(self.connect_server.url, app_id, app_mode) + app_id, existing_app_mode = app_store.resolve(self.remote_server.url, app_id, app_mode) logger.debug("Using app mode from app %s: %s" % (app_id, app_mode)) elif app_id is not None: # Don't read app metadata if app-id is specified. Instead, we need # to get this from the remote. - if isinstance(self.connect_server, RSConnectServer): - app = get_app_info(self.connect_server, app_id) + if isinstance(self.remote_server, RSConnectServer): + app = get_app_info(self.remote_server, app_id) existing_app_mode = AppModes.get_by_ordinal(app.get("app_mode", 0), True) else: - app = get_shinyapp_info(self.connect_server, app_id) + app = get_shinyapp_info(self.remote_server, app_id) existing_app_mode = AppModes.get_by_cloud_name(app.json_data["mode"]) if existing_app_mode and app_mode != existing_app_mode: msg = ( @@ -778,7 +778,7 @@ def validate_app_mode(self, *args, **kwargs): def server_settings(self): try: result = self.client.server_settings() - self.connect_server.handle_bad_response(result) + self.remote_server.handle_bad_response(result) except SSLError as ssl_error: raise RSConnectException("There is an SSL/TLS configuration problem: %s" % ssl_error) return result @@ -789,7 +789,7 @@ def verify_api_key(self, server=None): If the API key verifies, we return the username of the associated user. """ if not server: - server = self.connect_server + server = self.remote_server if isinstance(server, ShinyappsServer): raise RSConnectException("Shinnyapps server does not use an API key.") with RSConnectClient(server) as client: @@ -803,7 +803,7 @@ def verify_api_key(self, server=None): @property def api_username(self): result = self.client.me() - self.connect_server.handle_bad_response(result) + self.remote_server.handle_bad_response(result) return result["username"] @property @@ -815,7 +815,7 @@ def python_info(self): :return: the Python installation information from Connect. """ result = self.client.python_settings() - self.connect_server.handle_bad_response(result) + self.remote_server.handle_bad_response(result) return result @property @@ -875,7 +875,7 @@ def make_deployment_name(self, title, force_unique): # Now, make sure it's unique, if needed. if force_unique: - name = find_unique_name(self.connect_server, name) + name = find_unique_name(self.remote_server, name) return name From 8f27ecc280ce0009dd64a5a03f262e30287c64d3 Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Thu, 30 Jun 2022 15:13:39 -0400 Subject: [PATCH 20/57] account_name is from server_data.account --- rsconnect/api.py | 1 + 1 file changed, 1 insertion(+) diff --git a/rsconnect/api.py b/rsconnect/api.py index ea052f7f..43487794 100644 --- a/rsconnect/api.py +++ b/rsconnect/api.py @@ -396,6 +396,7 @@ def setup_remote_server( api_key = server_data.api_key insecure = server_data.insecure ca_data = server_data.ca_data + account_name = server_data.account token = server_data.token secret = server_data.secret From 3e4b0a7ab31ba41cc5b070522be0cc4bfd2efc49 Mon Sep 17 00:00:00 2001 From: Bincheng Wu Date: Thu, 30 Jun 2022 15:15:09 -0400 Subject: [PATCH 21/57] remove target from server add --- rsconnect/main.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/rsconnect/main.py b/rsconnect/main.py index f2617235..79be8033 100644 --- a/rsconnect/main.py +++ b/rsconnect/main.py @@ -324,7 +324,6 @@ def add(name, server, api_key, insecure, cacert, account, token, secret, verbose server_store.set( name, - "shinyapps", shinyapps_server.url, account=shinyapps_server.account_name, token=shinyapps_server.token, @@ -340,7 +339,6 @@ def add(name, server, api_key, insecure, cacert, account, token, secret, verbose server_store.set( name, - "connect", real_server.url, real_server.api_key, real_server.insecure, From d2a4370dc88ed8bf7f64919727000eab21969a34 Mon Sep 17 00:00:00 2001 From: Matthew Lynch Date: Thu, 30 Jun 2022 14:27:17 -0500 Subject: [PATCH 22/57] implement getting app mode from shinyapps.io --- rsconnect/actions.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rsconnect/actions.py b/rsconnect/actions.py index 0926a6a1..9438dd76 100644 --- a/rsconnect/actions.py +++ b/rsconnect/actions.py @@ -1537,8 +1537,8 @@ def _gather_basic_deployment_info_for_framework( app = api.get_app_info(remote_server, app_id) existing_app_mode = AppModes.get_by_ordinal(app.get("app_mode", 0), True) else: - # TODO this - raise NotImplementedError("TODO (mslynch): get app mode from lucid-server") + app = api.get_shinyapp_info(remote_server, app_id) + existing_app_mode = AppModes.get_by_cloud_name(app.json_data["mode"]) if existing_app_mode and app_mode != existing_app_mode: msg = ( "Deploying with mode '%s',\n" From 556f6ec73f899f052d81479d5a8b0e9cf032b03d Mon Sep 17 00:00:00 2001 From: Matthew Lynch Date: Thu, 30 Jun 2022 15:22:13 -0500 Subject: [PATCH 23/57] remove redundant lines --- rsconnect/api.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/rsconnect/api.py b/rsconnect/api.py index 2b477728..51b62791 100644 --- a/rsconnect/api.py +++ b/rsconnect/api.py @@ -493,8 +493,6 @@ def validate_connect_server( ca_data = None if cacert: ca_data = text_type(cacert.read()) - api_key = api_key or self.remote_server.api_key - insecure = insecure or self.remote_server.insecure if not ca_data: ca_data = self.remote_server.ca_data From 7925cb060a0a164b1e4b9fad91755c7af000c642 Mon Sep 17 00:00:00 2001 From: Matthew Lynch Date: Thu, 30 Jun 2022 15:48:29 -0500 Subject: [PATCH 24/57] tweak connect server validation --- rsconnect/api.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/rsconnect/api.py b/rsconnect/api.py index 51b62791..ec27a81e 100644 --- a/rsconnect/api.py +++ b/rsconnect/api.py @@ -472,6 +472,7 @@ def validate_server( return self + def validate_connect_server( self, name: str = None, @@ -484,6 +485,15 @@ def validate_connect_server( ): """ Validate that the user gave us enough information to talk to shinyapps.io or a Connect server. + :param name: the nickname, if any, specified by the user. + :param url: the URL, if any, specified by the user. + :param api_key: the API key, if any, specified by the user. + :param insecure: a flag noting whether TLS host/validation should be skipped. + :param cacert: the file object of a CA certs file containing certificates to use. + :param api_key_is_required: a flag that notes whether the API key is required or may + be omitted. + :param token: The shinyapps.io authentication token. + :param secret: The shinyapps.io authentication secret. """ url = url or self.remote_server.url api_key = api_key or self.remote_server.api_key @@ -493,6 +503,8 @@ def validate_connect_server( ca_data = None if cacert: ca_data = text_type(cacert.read()) + api_key = api_key or self.remote_server.api_key + insecure = insecure or self.remote_server.insecure if not ca_data: ca_data = self.remote_server.ca_data From 4c87cc526c96ad2a140dde6212824b4e313be3fb Mon Sep 17 00:00:00 2001 From: Matthew Lynch Date: Thu, 30 Jun 2022 16:38:00 -0500 Subject: [PATCH 25/57] implement auto naming for shinyapps --- rsconnect/actions.py | 7 +------ rsconnect/api.py | 41 +++++++++++++++++++++++++++++++---------- 2 files changed, 32 insertions(+), 16 deletions(-) diff --git a/rsconnect/actions.py b/rsconnect/actions.py index 9438dd76..56403a31 100644 --- a/rsconnect/actions.py +++ b/rsconnect/actions.py @@ -347,12 +347,7 @@ def _make_deployment_name(remote_server: api.RemoteServer, title: str, force_uni # Now, make sure it's unique, if needed. if force_unique: - if isinstance(remote_server, api.RSConnectServer): - name = api.find_unique_name(remote_server, name) - else: - # TODO (mslynch): is this necessary to check? - pass - # raise NotImplementedError("TODO (mslynch): find unique name from lucid-server") + name = api.find_unique_name(remote_server, name) return name diff --git a/rsconnect/api.py b/rsconnect/api.py index ec27a81e..942173ad 100644 --- a/rsconnect/api.py +++ b/rsconnect/api.py @@ -573,7 +573,7 @@ def make_bundle(self, func: Callable, *args, **kwargs): d = self.state d["title_is_default"] = not bool(title) d["title"] = title or _default_title(path) - force_unique_name = app_id is None and isinstance(self.remote_server, RSConnectServer) + force_unique_name = app_id is None d["deployment_name"] = self.make_deployment_name(d["title"], force_unique_name) try: @@ -604,7 +604,6 @@ def check_server_capabilities(self, capability_functions): :param details_source: the source for obtaining server details, gather_server_details(), by default. """ - # TODO (mslynch): check shinyapps capabilities? if isinstance(self.remote_server, ShinyappsServer): return self @@ -884,7 +883,6 @@ def make_deployment_name(self, title, force_unique): that we collapse repeating underscores and, if the name is too short, it is padded to the left with underscores. - :param connect_server: the information needed to interact with the Connect server. :param title: the title to start with. :param force_unique: a flag noting whether the generated name must be forced to be unique. @@ -982,6 +980,9 @@ def create_application(self, account_id, application_name): def get_accounts(self): return self.get("/v1/accounts/") + def _get_applications_like_name_page(self, name: str, offset: int): + return self.get("/v1/applications?filter=name:like:{}&offset={}&count=100&use_advanced_filters=true".format(name, offset)) + def create_bundle(self, application_id: int, content_type: str, content_length: int, checksum: str): bundle_data = { "application": application_id, @@ -1061,6 +1062,22 @@ def do_deploy(self, bundle_id, app_id): self._server.handle_bad_response(deploy_task) self.wait_until_task_is_successful(deploy_task.json_data["id"]) + def get_applications_like_name(self, name): + applications = [] + + results = self._get_applications_like_name_page(name, 0) + self._server.handle_bad_response(results) + offset = 0 + + while len(applications) < int(results.json_data['total']): + results = self._get_applications_like_name_page(name, offset) + self._server.handle_bad_response(results) + applications = results.json_data['applications'] + applications.extend(applications) + offset += int(results.json_data['count']) + + return [app['name'] for app in applications] + def verify_server(connect_server): """ @@ -1322,20 +1339,24 @@ def mapping_filter(client, app): return apps -def find_unique_name(connect_server, name): +def find_unique_name(remote_server: AbstractRemoteServer, name: str): """ Poll through existing apps to see if anything with a similar name exists. If so, start appending numbers until a unique name is found. - :param connect_server: the Connect server information. + :param remote_server: the remote server information. :param name: the default name for an app. :return: the name, potentially with a suffixed number to guarantee uniqueness. """ - existing_names = retrieve_matching_apps( - connect_server, - filters={"search": name}, - mapping_function=lambda client, app: app["name"], - ) + if isinstance(remote_server, RSConnectServer): + existing_names = retrieve_matching_apps( + remote_server, + filters={"search": name}, + mapping_function=lambda client, app: app["name"], + ) + else: + client = ShinyappsClient(remote_server) + existing_names = client.get_applications_like_name(name) if name in existing_names: suffix = 1 From 64b114119cb853deadf93aab8422978ebf33c9da Mon Sep 17 00:00:00 2001 From: Matthew Lynch Date: Fri, 1 Jul 2022 11:31:58 -0500 Subject: [PATCH 26/57] use mock for adding shinyapps credential test --- .gitignore | 1 + requirements.txt | 1 + tests/test_main.py | 58 +++++++++++++++++++++++++++++++--------------- 3 files changed, 41 insertions(+), 19 deletions(-) diff --git a/.gitignore b/.gitignore index ff527bc2..99be961e 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,7 @@ /rsconnect/version.py htmlcov /tests/testdata/**/rsconnect-python/ +/tests/test-home/ /docs/docs/index.md /docs/docs/changelog.md /rsconnect-build diff --git a/requirements.txt b/requirements.txt index 5a53544e..1baec990 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,6 +3,7 @@ click>=7.0.0 coverage flake8 funcsigs +httpretty==1.1.4 importlib-metadata ipykernel ipython diff --git a/tests/test_main.py b/tests/test_main.py index 18ee081d..9071c54c 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -1,8 +1,11 @@ import os +import shutil import unittest from os.path import join from unittest import TestCase + +import httpretty from click.testing import CliRunner from .utils import ( @@ -20,6 +23,10 @@ class TestMain(TestCase): + def setUp(self): + shutil.rmtree('test-home', ignore_errors=True) + os.environ['HOME'] = 'test-home' + def require_connect(self): connect_server = os.environ.get("CONNECT_SERVER", None) if connect_server is None: @@ -104,26 +111,39 @@ def test_add_connect(self): self.assertEqual(result.exit_code, 0, result.output) self.assertIn("OK", result.output) - # TODO (mslynch): mock shinyapps.io - @unittest.skip + # @unittest.skip + @httpretty.activate(verbose=True, allow_net_connect=False) def test_add_shinyapps(self): - runner = CliRunner() - result = runner.invoke( - cli, - [ - "add", - "--target", - "shinyapps", - "--name", - "my-shinyapps", - "--token", - "someToken", - "--secret", - "c29tZVNlY3JldAo=", - ], - ) - self.assertEqual(result.exit_code, 0, result.output) - self.assertIn("shinyapps.io credential", result.output) + original_api_key_value = os.environ.pop("CONNECT_API_KEY") + + try: + httpretty.register_uri( + httpretty.GET, + "http://localhost:3939/v1/users/me", + body='{"id": 1000}', + status=200 + ) + + runner = CliRunner() + result = runner.invoke( + cli, + [ + "add", + "--account", + "someAccount", + "--name", + "my-shinyapps", + "--token", + "someToken", + "--secret", + "c29tZVNlY3JldAo=", + ], + ) + self.assertEqual(result.exit_code, 0, result.output) + self.assertIn("shinyapps.io credential", result.output) + + finally: + os.environ["CONNECT_API_KEY"] = original_api_key_value def test_add_shinyapps_missing_options(self): original_api_key_value = os.environ.pop("CONNECT_API_KEY") From 13e64c31c57b5de3c8903d7304ec0f2b84c80fe7 Mon Sep 17 00:00:00 2001 From: Matthew Lynch Date: Fri, 1 Jul 2022 11:39:11 -0500 Subject: [PATCH 27/57] format and linting fixes --- rsconnect/api.py | 39 +++++++++------------------------------ rsconnect/validation.py | 4 ---- tests/test_main.py | 12 +++--------- tests/test_metadata.py | 3 +-- 4 files changed, 13 insertions(+), 45 deletions(-) diff --git a/rsconnect/api.py b/rsconnect/api.py index 942173ad..147780bb 100644 --- a/rsconnect/api.py +++ b/rsconnect/api.py @@ -1,14 +1,9 @@ """ RStudio Connect API client and utility functions """ -from os.path import abspath, basename +from os.path import abspath import time from typing import IO, Callable -from _ssl import SSLError -import re -from warnings import warn -from six import text_type -import gc import base64 import datetime import hashlib @@ -18,7 +13,6 @@ from _ssl import SSLError from urllib import parse from urllib.parse import urlparse -import click import re @@ -27,7 +21,6 @@ import gc from . import validation -from .bundle import fake_module_file_from_directory from .http_support import HTTPResponse, HTTPServer, append_to_path, CookieJar from .log import logger, connect_logger, cls_logged, console_logger from .models import AppModes @@ -400,7 +393,7 @@ def setup_remote_server( token: str = None, secret: str = None, ): - target = validation.validate_connection_options( + validation.validate_connection_options( name=name, url=url, api_key=api_key, @@ -472,7 +465,6 @@ def validate_server( return self - def validate_connect_server( self, name: str = None, @@ -981,7 +973,9 @@ def get_accounts(self): return self.get("/v1/accounts/") def _get_applications_like_name_page(self, name: str, offset: int): - return self.get("/v1/applications?filter=name:like:{}&offset={}&count=100&use_advanced_filters=true".format(name, offset)) + return self.get( + "/v1/applications?filter=name:like:{}&offset={}&count=100&use_advanced_filters=true".format(name, offset) + ) def create_bundle(self, application_id: int, content_type: str, content_length: int, checksum: str): bundle_data = { @@ -1069,14 +1063,14 @@ def get_applications_like_name(self, name): self._server.handle_bad_response(results) offset = 0 - while len(applications) < int(results.json_data['total']): + while len(applications) < int(results.json_data["total"]): results = self._get_applications_like_name_page(name, offset) self._server.handle_bad_response(results) - applications = results.json_data['applications'] + applications = results.json_data["applications"] applications.extend(applications) - offset += int(results.json_data['count']) + offset += int(results.json_data["count"]) - return [app['name'] for app in applications] + return [app["name"] for app in applications] def verify_server(connect_server): @@ -1387,18 +1381,3 @@ def _to_server_check_list(url): items = ["%s"] return [item % url for item in items] - - -def _default_title(file_name): - """ - Produce a default content title from the given file path. The result is - guaranteed to be between 3 and 1024 characters long, as required by RStudio - Connect. - - :param file_name: the name from which the title will be derived. - :return: the derived title. - """ - # Make sure we have enough of a path to derive text from. - file_name = abspath(file_name) - # noinspection PyTypeChecker - return basename(file_name).rsplit(".", 1)[0][:1024].rjust(3, "0") diff --git a/rsconnect/validation.py b/rsconnect/validation.py index 609f3993..d849d6e9 100644 --- a/rsconnect/validation.py +++ b/rsconnect/validation.py @@ -44,7 +44,3 @@ def validate_connection_options(name, url, api_key, insecure, cacert, account, t raise api.RSConnectException( "-a/--account, -T/--token, and -S/--secret must all be provided for shinyapps.io." ) - - return "shinyapps" - - return "connect" diff --git a/tests/test_main.py b/tests/test_main.py index 9071c54c..6aad3df1 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -1,6 +1,5 @@ import os import shutil -import unittest from os.path import join from unittest import TestCase @@ -24,8 +23,8 @@ class TestMain(TestCase): def setUp(self): - shutil.rmtree('test-home', ignore_errors=True) - os.environ['HOME'] = 'test-home' + shutil.rmtree("test-home", ignore_errors=True) + os.environ["HOME"] = "test-home" def require_connect(self): connect_server = os.environ.get("CONNECT_SERVER", None) @@ -117,12 +116,7 @@ def test_add_shinyapps(self): original_api_key_value = os.environ.pop("CONNECT_API_KEY") try: - httpretty.register_uri( - httpretty.GET, - "http://localhost:3939/v1/users/me", - body='{"id": 1000}', - status=200 - ) + httpretty.register_uri(httpretty.GET, "http://localhost:3939/v1/users/me", body='{"id": 1000}', status=200) runner = CliRunner() result = runner.invoke( diff --git a/tests/test_metadata.py b/tests/test_metadata.py index 063cebbb..40823384 100644 --- a/tests/test_metadata.py +++ b/tests/test_metadata.py @@ -113,8 +113,7 @@ def test_resolve_by_url(self): def test_resolve_by_default(self): # with multiple entries, server None will not resolve by default - name, server, api_key, insecure, ca_cert = None, None, None, None, None - server_data = self.server_store.resolve(name, server) + server_data = self.server_store.resolve(None, None) self.assertEqual(server_data.url, None) # with only a single entry, server None will resolve to that entry From b6b7e561889fbf88c52bb77ec121a2b43c2ab3e2 Mon Sep 17 00:00:00 2001 From: Matthew Lynch Date: Fri, 1 Jul 2022 11:49:18 -0500 Subject: [PATCH 28/57] type checking fixes --- rsconnect/actions.py | 8 ++++---- rsconnect/api.py | 18 ++++-------------- 2 files changed, 8 insertions(+), 18 deletions(-) diff --git a/rsconnect/actions.py b/rsconnect/actions.py index 56403a31..8fc078a9 100644 --- a/rsconnect/actions.py +++ b/rsconnect/actions.py @@ -322,7 +322,7 @@ def check_server_capabilities(connect_server, capability_functions, details_sour raise RSConnectException(message) -def _make_deployment_name(remote_server: api.RemoteServer, title: str, force_unique: bool) -> str: +def _make_deployment_name(remote_server: api.TargetableServer, title: str, force_unique: bool) -> str: """ Produce a name for a deployment based on its title. It is assumed that the title is already defaulted and validated as appropriate (meaning the title @@ -1456,7 +1456,7 @@ def _generate_gather_basic_deployment_info_for_python(app_mode: AppMode) -> typi """ def gatherer( - remote_server: api.RemoteServer, + remote_server: api.TargetableServer, app_store: AppStore, directory: str, entry_point: str, @@ -1486,7 +1486,7 @@ def gatherer( def _gather_basic_deployment_info_for_framework( - remote_server: api.RemoteServer, + remote_server: api.TargetableServer, app_store: AppStore, directory: str, entry_point: str, @@ -1710,7 +1710,7 @@ def create_quarto_deployment_bundle( def deploy_bundle( - remote_server: api.RemoteServer, + remote_server: api.TargetableServer, app_id: int, name: str, title: str, diff --git a/rsconnect/api.py b/rsconnect/api.py index 147780bb..1cafb80c 100644 --- a/rsconnect/api.py +++ b/rsconnect/api.py @@ -30,15 +30,6 @@ class AbstractRemoteServer: - # @property - # @abc.abstractmethod - # def url(self) -> str: - # pass - # - # @property - # @abc.abstractmethod - # def remote_name(self) -> str: - # pass url: str remote_name: str @@ -97,6 +88,7 @@ def __init__(self, url, api_key, insecure=False, ca_data=None): # This is specifically not None. self.cookie_jar = CookieJar() +TargetableServer = typing.Union[ShinyappsServer, RSConnectServer] class S3Server(AbstractRemoteServer): remote_name = "S3" @@ -105,8 +97,6 @@ def __init__(self, url: str): self.url = url -RemoteServer = typing.Union[ShinyappsServer, RSConnectServer] - class RSConnectClient(HTTPServer): def __init__(self, server: RSConnectServer, cookies=None, timeout=30): @@ -1034,10 +1024,10 @@ def prepare_deploy(self, app_id: typing.Optional[str], app_name: str, bundle_siz else: application = self.get_application(app_id) self._server.handle_bad_response(application) - app_id = application.json_data["id"] + app_id_int = application.json_data["id"] app_url = application.json_data["url"] - bundle = self.create_bundle(app_id, "application/x-tar", bundle_size, bundle_hash) + bundle = self.create_bundle(app_id_int, "application/x-tar", bundle_size, bundle_hash) self._server.handle_bad_response(bundle) return PrepareDeployResult( @@ -1333,7 +1323,7 @@ def mapping_filter(client, app): return apps -def find_unique_name(remote_server: AbstractRemoteServer, name: str): +def find_unique_name(remote_server: TargetableServer, name: str): """ Poll through existing apps to see if anything with a similar name exists. If so, start appending numbers until a unique name is found. From 2c8e15f74209c0c05deeec06a692aad77882a6c2 Mon Sep 17 00:00:00 2001 From: Matthew Lynch Date: Fri, 1 Jul 2022 11:51:19 -0500 Subject: [PATCH 29/57] type checking fixes --- rsconnect/validation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rsconnect/validation.py b/rsconnect/validation.py index d849d6e9..5e5085e8 100644 --- a/rsconnect/validation.py +++ b/rsconnect/validation.py @@ -3,7 +3,7 @@ from . import api -def _get_present_options(options: dict[str, typing.Optional[str]]) -> list[str]: +def _get_present_options(options: typing.Dict[str, typing.Optional[str]]) -> typing.List[str]: return [k for k, v in options.items() if v] From 380974dc44ec5999b1496505bc6eff34c9599f91 Mon Sep 17 00:00:00 2001 From: Matthew Lynch Date: Fri, 1 Jul 2022 11:59:59 -0500 Subject: [PATCH 30/57] python 3.5 compatibility --- rsconnect/api.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/rsconnect/api.py b/rsconnect/api.py index 1cafb80c..6b17215a 100644 --- a/rsconnect/api.py +++ b/rsconnect/api.py @@ -14,7 +14,6 @@ from urllib import parse from urllib.parse import urlparse - import re from warnings import warn from six import text_type @@ -30,8 +29,9 @@ class AbstractRemoteServer: - url: str - remote_name: str + def __init__(self, url: str, remote_name: str): + self.url = url + self.remote_name = remote_name def handle_bad_response(self, response): if isinstance(response, HTTPResponse): @@ -63,10 +63,8 @@ class ShinyappsServer(AbstractRemoteServer): instance of the shinyapps.io server. """ - remote_name = "shinyapps.io" - def __init__(self, url: str, account_name: str, token: str, secret: str): - self.url = url + super().__init__(url, "shinyapps.io") self.account_name = account_name self.token = token self.secret = secret @@ -78,18 +76,18 @@ class RSConnectServer(AbstractRemoteServer): instance of the Connect server. """ - remote_name = "RStudio Connect" - def __init__(self, url, api_key, insecure=False, ca_data=None): - self.url = url + super().__init__(url, "RStudio Connect") self.api_key = api_key self.insecure = insecure self.ca_data = ca_data # This is specifically not None. self.cookie_jar = CookieJar() + TargetableServer = typing.Union[ShinyappsServer, RSConnectServer] + class S3Server(AbstractRemoteServer): remote_name = "S3" @@ -97,7 +95,6 @@ def __init__(self, url: str): self.url = url - class RSConnectClient(HTTPServer): def __init__(self, server: RSConnectServer, cookies=None, timeout=30): if cookies is None: From 9049966d605660a1b7a76e90a54fcd106a442b11 Mon Sep 17 00:00:00 2001 From: Matthew Lynch Date: Fri, 1 Jul 2022 12:01:35 -0500 Subject: [PATCH 31/57] python 3.5 compatibility --- rsconnect/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rsconnect/api.py b/rsconnect/api.py index 6b17215a..1e0dcb29 100644 --- a/rsconnect/api.py +++ b/rsconnect/api.py @@ -531,7 +531,7 @@ def validate_shinyapps_server( result = client.get_current_user() server.handle_bad_response(result) except RSConnectException as exc: - raise RSConnectException(f"Failed to verify with shinyapps.io ({str(exc)}).") + raise RSConnectException("Failed to verify with shinyapps.io ({}).".format(exc)) @cls_logged("Making bundle ...") def make_bundle(self, func: Callable, *args, **kwargs): From d107182ae62476e403599b4b0c4c4dbce48e0513 Mon Sep 17 00:00:00 2001 From: Matthew Lynch Date: Fri, 1 Jul 2022 12:03:24 -0500 Subject: [PATCH 32/57] python 3.5 compatibility --- rsconnect/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rsconnect/api.py b/rsconnect/api.py index 1e0dcb29..7316f920 100644 --- a/rsconnect/api.py +++ b/rsconnect/api.py @@ -706,7 +706,7 @@ def emit_task_log( @cls_logged("Saving deployed information...") def save_deployed_info(self, *args, **kwargs): - app_store: AppStore = self.get("app_store", *args, **kwargs) + app_store = self.get("app_store", *args, **kwargs) path = ( self.get("path", **kwargs) or self.get("file", **kwargs) From 41312964274f027a962e3f18e2f6aa5b88c8def2 Mon Sep 17 00:00:00 2001 From: Matthew Lynch Date: Fri, 1 Jul 2022 12:09:41 -0500 Subject: [PATCH 33/57] fix test_add_shinyapps and test_add_shinyapps_missing_options for missing environment variables --- tests/test_main.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/test_main.py b/tests/test_main.py index 6aad3df1..7b50ea35 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -113,8 +113,7 @@ def test_add_connect(self): # @unittest.skip @httpretty.activate(verbose=True, allow_net_connect=False) def test_add_shinyapps(self): - original_api_key_value = os.environ.pop("CONNECT_API_KEY") - + original_api_key_value = os.environ.pop("CONNECT_API_KEY", None) try: httpretty.register_uri(httpretty.GET, "http://localhost:3939/v1/users/me", body='{"id": 1000}', status=200) @@ -137,7 +136,8 @@ def test_add_shinyapps(self): self.assertIn("shinyapps.io credential", result.output) finally: - os.environ["CONNECT_API_KEY"] = original_api_key_value + if original_api_key_value: + os.environ["CONNECT_API_KEY"] = original_api_key_value def test_add_shinyapps_missing_options(self): original_api_key_value = os.environ.pop("CONNECT_API_KEY") @@ -158,4 +158,5 @@ def test_add_shinyapps_missing_options(self): str(result.exception), "--account, --token, and --secret must all be provided for shinyapps.io." ) finally: - os.environ["CONNECT_API_KEY"] = original_api_key_value + if original_api_key_value: + os.environ["CONNECT_API_KEY"] = original_api_key_value From 3d159e1bc15263b1fac3442c93d554ea0de1d876 Mon Sep 17 00:00:00 2001 From: Matthew Lynch Date: Fri, 1 Jul 2022 12:29:18 -0500 Subject: [PATCH 34/57] fix shinyapps tests for possibly-present environment variables --- tests/test_main.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tests/test_main.py b/tests/test_main.py index 7b50ea35..83c083f5 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -114,8 +114,9 @@ def test_add_connect(self): @httpretty.activate(verbose=True, allow_net_connect=False) def test_add_shinyapps(self): original_api_key_value = os.environ.pop("CONNECT_API_KEY", None) + original_server_value = os.environ.pop("CONNECT_SERVER", None) try: - httpretty.register_uri(httpretty.GET, "http://localhost:3939/v1/users/me", body='{"id": 1000}', status=200) + httpretty.register_uri(httpretty.GET, "https://api.shinyapps.io/v1/users/me", body='{"id": 1000}', status=200) runner = CliRunner() result = runner.invoke( @@ -138,9 +139,13 @@ def test_add_shinyapps(self): finally: if original_api_key_value: os.environ["CONNECT_API_KEY"] = original_api_key_value + if original_server_value: + os.environ["CONNECT_SERVER"] = original_server_value + def test_add_shinyapps_missing_options(self): - original_api_key_value = os.environ.pop("CONNECT_API_KEY") + original_api_key_value = os.environ.pop("CONNECT_API_KEY", None) + original_server_value = os.environ.pop("CONNECT_SERVER", None) try: runner = CliRunner() result = runner.invoke( @@ -160,3 +165,5 @@ def test_add_shinyapps_missing_options(self): finally: if original_api_key_value: os.environ["CONNECT_API_KEY"] = original_api_key_value + if original_server_value: + os.environ["CONNECT_SERVER"] = original_server_value From 371ec5d082afad10470d7c218fa7538784d68ba0 Mon Sep 17 00:00:00 2001 From: Matthew Lynch Date: Fri, 1 Jul 2022 12:34:37 -0500 Subject: [PATCH 35/57] fix formatting --- tests/test_main.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/test_main.py b/tests/test_main.py index 83c083f5..628275ea 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -116,7 +116,9 @@ def test_add_shinyapps(self): original_api_key_value = os.environ.pop("CONNECT_API_KEY", None) original_server_value = os.environ.pop("CONNECT_SERVER", None) try: - httpretty.register_uri(httpretty.GET, "https://api.shinyapps.io/v1/users/me", body='{"id": 1000}', status=200) + httpretty.register_uri( + httpretty.GET, "https://api.shinyapps.io/v1/users/me", body='{"id": 1000}', status=200 + ) runner = CliRunner() result = runner.invoke( @@ -142,7 +144,6 @@ def test_add_shinyapps(self): if original_server_value: os.environ["CONNECT_SERVER"] = original_server_value - def test_add_shinyapps_missing_options(self): original_api_key_value = os.environ.pop("CONNECT_API_KEY", None) original_server_value = os.environ.pop("CONNECT_SERVER", None) From c3babe866f21028e18fd662e7f9f91a6e227d658 Mon Sep 17 00:00:00 2001 From: Matthew Lynch Date: Fri, 1 Jul 2022 13:24:49 -0500 Subject: [PATCH 36/57] refactor validation --- rsconnect/main.py | 27 +++++++++++---------------- rsconnect/validation.py | 11 +++++++---- tests/test_main.py | 4 ++-- 3 files changed, 20 insertions(+), 22 deletions(-) diff --git a/rsconnect/main.py b/rsconnect/main.py index 9c6c3dd1..a0a2ffc0 100644 --- a/rsconnect/main.py +++ b/rsconnect/main.py @@ -35,7 +35,7 @@ emit_build_log, ) -from . import api, VERSION +from . import api, VERSION, validation from .api import RSConnectExecutor, filter_out_server_info from .bundle import ( are_apis_supported_on_server, @@ -303,24 +303,19 @@ def add(name, server, api_key, insecure, cacert, account, token, secret, verbose set_verbosity(verbose) - connect_options = {"--api-key": api_key, "--insecure": insecure, "--cacert": cacert} - shinyapps_options = {"--token": token, "--secret": secret, "--account": account} - - present_connect_options = [k for k, v in connect_options.items() if v] - present_shinyapps_options = [k for k, v in shinyapps_options.items() if v] - - if present_connect_options and present_shinyapps_options: - raise api.RSConnectException( - "Connect options ({}) may not be passed alongside shinyapps.io options ({}).".format( - ", ".join(present_connect_options), ", ".join(present_shinyapps_options) - ) - ) + validation.validate_new_server_options( + url=server, + api_key=api_key, + insecure=insecure, + cacert=cacert, + account=account, + token=token, + secret=secret, + ) old_server = server_store.get_by_name(name) - if present_shinyapps_options: - if len(present_shinyapps_options) != 3: - raise api.RSConnectException("--account, --token, and --secret must all be provided for shinyapps.io.") + if account: shinyapps_server = api.ShinyappsServer(server or "https://api.shinyapps.io", account, token, secret) _test_shinyappsio_creds(shinyapps_server) diff --git a/rsconnect/validation.py b/rsconnect/validation.py index 5e5085e8..747d1113 100644 --- a/rsconnect/validation.py +++ b/rsconnect/validation.py @@ -7,6 +7,10 @@ def _get_present_options(options: typing.Dict[str, typing.Optional[str]]) -> typ return [k for k, v in options.items() if v] +def validate_new_server_options(url, api_key, insecure, cacert, account, token, secret): + validate_connection_options(None, url, api_key, insecure, cacert, account, token, secret) + + def validate_connection_options(name, url, api_key, insecure, cacert, account, token, secret): """ Validates provided Connect or shinyapps.io connection options and returns which target to use given the provided @@ -14,8 +18,7 @@ def validate_connection_options(name, url, api_key, insecure, cacert, account, t """ connect_options = {"-k/--api-key": api_key, "-i/--insecure": insecure, "-c/--cacert": cacert} shinyapps_options = {"-T/--token": token, "-S/--secret": secret, "-a/--account": account} - options_mutually_exclusive_with_name = {"--server": url, **connect_options, **shinyapps_options} - + options_mutually_exclusive_with_name = {"-s/--server": url, **connect_options, **shinyapps_options} present_options_mutually_exclusive_with_name = _get_present_options(options_mutually_exclusive_with_name) if name and present_options_mutually_exclusive_with_name: @@ -24,9 +27,9 @@ def validate_connection_options(name, url, api_key, insecure, cacert, account, t ", ".join(present_options_mutually_exclusive_with_name) ) ) - if not name and not url: + if not name and not url and not shinyapps_options: raise api.RSConnectException( - "You must specify one of -n/--name OR -s/--server OR -a/--account, -T/--token, -S/--secret." + "You must specify one of -n/--name OR -s/--server OR -a/--account, -T/--token, -S/--secret." ) present_connect_options = _get_present_options(connect_options) diff --git a/tests/test_main.py b/tests/test_main.py index 628275ea..ffb6cf19 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -110,7 +110,6 @@ def test_add_connect(self): self.assertEqual(result.exit_code, 0, result.output) self.assertIn("OK", result.output) - # @unittest.skip @httpretty.activate(verbose=True, allow_net_connect=False) def test_add_shinyapps(self): original_api_key_value = os.environ.pop("CONNECT_API_KEY", None) @@ -161,7 +160,8 @@ def test_add_shinyapps_missing_options(self): ) self.assertEqual(result.exit_code, 1, result.output) self.assertEqual( - str(result.exception), "--account, --token, and --secret must all be provided for shinyapps.io." + str(result.exception), + "-a/--account, -T/--token, and -S/--secret must all be provided for shinyapps.io.", ) finally: if original_api_key_value: From 7658448d8301da7f18bb734e3929c75b593f3f01 Mon Sep 17 00:00:00 2001 From: Matthew Lynch Date: Fri, 1 Jul 2022 13:25:57 -0500 Subject: [PATCH 37/57] remove extraneous file --- .../connect_remote_6443.json | 83 ------------------- 1 file changed, 83 deletions(-) delete mode 100644 tests/rsconnect-build-test/connect_remote_6443.json diff --git a/tests/rsconnect-build-test/connect_remote_6443.json b/tests/rsconnect-build-test/connect_remote_6443.json deleted file mode 100644 index 32f7e05c..00000000 --- a/tests/rsconnect-build-test/connect_remote_6443.json +++ /dev/null @@ -1,83 +0,0 @@ -{ - "rsconnect_build_running": false, - "rsconnect_content": { - "c96db3f3-87a1-4df5-9f58-eb109c397718": { - "guid": "c96db3f3-87a1-4df5-9f58-eb109c397718", - "bundle_id": "177", - "title": "orphan-proc-shiny-test", - "name": "orphan-proc-shiny-test", - "app_mode": "shiny", - "content_url": "https://connect.remote:6443/content/c96db3f3-87a1-4df5-9f58-eb109c397718/", - "dashboard_url": "https://connect.remote:6443/connect/#/apps/c96db3f3-87a1-4df5-9f58-eb109c397718", - "created_time": "2021-11-04T18:07:12Z", - "last_deployed_time": "2021-11-10T19:10:56Z", - "owner_guid": "edf26318-0027-4d9d-bbbb-54703ebb1855", - "rsconnect_build_status": "NEEDS_BUILD" - }, - "fe673896-f92a-40cc-be4c-e4872bb90a37": { - "guid": "fe673896-f92a-40cc-be4c-e4872bb90a37", - "bundle_id": "185", - "title": "interactive-rmd", - "name": "interactive-rmd", - "app_mode": "rmd-shiny", - "content_url": "https://connect.remote:6443/content/fe673896-f92a-40cc-be4c-e4872bb90a37/", - "dashboard_url": "https://connect.remote:6443/connect/#/apps/fe673896-f92a-40cc-be4c-e4872bb90a37", - "created_time": "2021-11-15T15:37:53Z", - "last_deployed_time": "2021-11-15T15:37:57Z", - "owner_guid": "edf26318-0027-4d9d-bbbb-54703ebb1855", - "rsconnect_build_status": "ERROR" - }, - "a0b6b5a2-5fbe-4293-8310-4f80054bc24f": { - "guid": "a0b6b5a2-5fbe-4293-8310-4f80054bc24f", - "bundle_id": "184", - "title": "stock-report-jupyter", - "name": "stock-report-jupyter", - "app_mode": "jupyter-static", - "content_url": "https://connect.remote:6443/content/a0b6b5a2-5fbe-4293-8310-4f80054bc24f/", - "dashboard_url": "https://connect.remote:6443/connect/#/apps/a0b6b5a2-5fbe-4293-8310-4f80054bc24f", - "created_time": "2021-11-15T15:27:18Z", - "last_deployed_time": "2021-11-15T15:35:27Z", - "owner_guid": "edf26318-0027-4d9d-bbbb-54703ebb1855", - "rsconnect_build_status": "RUNNING" - }, - "23315cc9-ed2a-40ad-9e99-e5e49066531a": { - "guid": "23315cc9-ed2a-40ad-9e99-e5e49066531a", - "bundle_id": "180", - "title": "static-rmd", - "name": "static-rmd2", - "app_mode": "rmd-static", - "content_url": "https://connect.remote:6443/content/23315cc9-ed2a-40ad-9e99-e5e49066531a/", - "dashboard_url": "https://connect.remote:6443/connect/#/apps/23315cc9-ed2a-40ad-9e99-e5e49066531a", - "created_time": "2021-11-15T15:20:58Z", - "last_deployed_time": "2021-11-15T15:25:31Z", - "owner_guid": "edf26318-0027-4d9d-bbbb-54703ebb1855", - "rsconnect_build_status": "COMPLETE", - "rsconnect_last_build_time": "2021-12-13T18:10:38Z", - "rsconnect_last_build_log": "/logs/localhost_3939/23315cc9-ed2a-40ad-9e99-e5e49066531a/ZUf44zVWHjODv1Rq.log", - "rsconnect_build_task_result": { - "id": "ZUf44zVWHjODv1Rq", - "user_id": 1, - "result": { - "type": "", - "data": null - }, - "finished": true, - "code": 0, - "error": "" - } - }, - "015143da-b75f-407c-81b1-99c4a724341e": { - "guid": "015143da-b75f-407c-81b1-99c4a724341e", - "bundle_id": "176", - "title": "plumber-async", - "name": "plumber-async", - "app_mode": "api", - "content_url": "https://connect.remote:6443/content/015143da-b75f-407c-81b1-99c4a724341e/", - "dashboard_url": "https://connect.remote:6443/connect/#/apps/015143da-b75f-407c-81b1-99c4a724341e", - "created_time": "2021-11-01T20:43:32Z", - "last_deployed_time": "2021-11-03T17:48:59Z", - "owner_guid": "edf26318-0027-4d9d-bbbb-54703ebb1855", - "rsconnect_build_status": "ERROR" - } - } -} \ No newline at end of file From 104710e12e366fd35675a3de040646d365e9379f Mon Sep 17 00:00:00 2001 From: Matthew Lynch Date: Fri, 1 Jul 2022 13:28:37 -0500 Subject: [PATCH 38/57] refactor kwarg calls in test for clarity --- tests/test_metadata.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_metadata.py b/tests/test_metadata.py index 40823384..ed15024c 100644 --- a/tests/test_metadata.py +++ b/tests/test_metadata.py @@ -141,7 +141,7 @@ def test_save_and_load(self): self.assertFalse(exists(path)) - server_store.set("foo", "http://connect.local", "connect", "notReallyAnApiKey", ca_data="/certs/connect") + server_store.set("foo", "http://connect.local", api_key="notReallyAnApiKey", ca_data="/certs/connect") self.assertTrue(exists(path)) @@ -268,8 +268,8 @@ def test_normalize_server_url(self): class TestBuildMetadata(TestCase): def setUp(self): self.server_store = ServerStore() - self.server_store.set("connect", "https://connect.remote:6443", "apiKey", insecure=True) - self.server = RSConnectServer("https://connect.remote:6443", "apiKey", True, None) + self.server_store.set("connect", "https://connect.remote:6443", api_key="apiKey", insecure=True) + self.server = RSConnectServer("https://connect.remote:6443", api_key="apiKey", insecure=True, ca_data=None) self.build_store = ContentBuildStore(self.server) self.build_store._set("rsconnect_build_running", False) self.build_store._set( From 30c5d7b3aad1b3d57bb03104ca03654af514e4fc Mon Sep 17 00:00:00 2001 From: Matthew Lynch Date: Tue, 5 Jul 2022 11:17:14 -0500 Subject: [PATCH 39/57] add mocked-http test for deploying shiny app to shinyapps.io --- .gitignore | 2 +- rsconnect/api.py | 10 +- rsconnect/http_support.py | 13 +- rsconnect/main.py | 2 +- .../connect_remote_6443.json | 83 ++++++++ tests/test_main.py | 168 ++++++++++++++- tests/test_metadata.py | 4 +- .../create-application.json | 124 +++++++++++ .../shinyapps-responses/create-bundle.json | 14 ++ .../shinyapps-responses/get-accounts.json | 198 ++++++++++++++++++ .../shinyapps-responses/get-applications.json | 6 + .../shinyapps-responses/get-bundle.json | 12 ++ .../shinyapps-responses/get-task.json | 17 ++ .../shinyapps-responses/get-user.json | 25 +++ .../shinyapps-responses/post-deploy.json | 3 + 15 files changed, 669 insertions(+), 12 deletions(-) create mode 100644 tests/rsconnect-build-test/connect_remote_6443.json create mode 100644 tests/testdata/shinyapps-responses/create-application.json create mode 100644 tests/testdata/shinyapps-responses/create-bundle.json create mode 100644 tests/testdata/shinyapps-responses/get-accounts.json create mode 100644 tests/testdata/shinyapps-responses/get-applications.json create mode 100644 tests/testdata/shinyapps-responses/get-bundle.json create mode 100644 tests/testdata/shinyapps-responses/get-task.json create mode 100644 tests/testdata/shinyapps-responses/get-user.json create mode 100644 tests/testdata/shinyapps-responses/post-deploy.json diff --git a/.gitignore b/.gitignore index 99be961e..a54596f9 100644 --- a/.gitignore +++ b/.gitignore @@ -18,7 +18,7 @@ /rsconnect/version.py htmlcov /tests/testdata/**/rsconnect-python/ -/tests/test-home/ +test-home/ /docs/docs/index.md /docs/docs/changelog.md /rsconnect-build diff --git a/rsconnect/api.py b/rsconnect/api.py index 7316f920..27b7ba07 100644 --- a/rsconnect/api.py +++ b/rsconnect/api.py @@ -64,7 +64,7 @@ class ShinyappsServer(AbstractRemoteServer): """ def __init__(self, url: str, account_name: str, token: str, secret: str): - super().__init__(url, "shinyapps.io") + super().__init__(url or "https://api.shinyapps.io", "shinyapps.io") self.account_name = account_name self.token = token self.secret = secret @@ -407,8 +407,6 @@ def setup_remote_server( if api_key: self.remote_server = RSConnectServer(url, api_key, insecure, ca_data) - elif token and secret: - self.remote_server = ShinyappsServer(url, account, token, secret) else: self.remote_server = ShinyappsServer(url, account, token, secret) @@ -971,7 +969,9 @@ def create_bundle(self, application_id: int, content_type: str, content_length: "content_length": content_length, "checksum": checksum, } - return self.post("/v1/bundles", body=bundle_data) + result = self.post("/v1/bundles", body=bundle_data) + print(result.json_data) + return result def set_bundle_status(self, bundle_id, bundle_status): return self.post("/v1/bundles/{}/status".format(bundle_id), body={"status": bundle_status}) @@ -1028,7 +1028,7 @@ def prepare_deploy(self, app_id: typing.Optional[str], app_name: str, bundle_siz self._server.handle_bad_response(bundle) return PrepareDeployResult( - int(app_id), + app_id_int, app_url, int(bundle.json_data["id"]), bundle.json_data["presigned_url"], diff --git a/rsconnect/http_support.py b/rsconnect/http_support.py index 52f6eaa4..ae53b62a 100644 --- a/rsconnect/http_support.py +++ b/rsconnect/http_support.py @@ -303,9 +303,18 @@ def _do_request( raise http.CannotSendRequest("Too many redirects") location = response.getheader("Location") - next_url = urljoin(self._url.geturl(), location) - logger.debug("--> Redirected to: %s" % next_url) + # Assume the redirect location will always be on the same domain. + if location.startswith("http"): + parsed_location = urlparse(location) + if parsed_location.query: + next_url = "{}?{}".format(parsed_location.path, parsed_location.query) + else: + next_url = parsed_location.path + else: + next_url = location + + logger.debug("--> Redirected to: %s" % urljoin(self._url.geturl(), location)) redirect_extra_headers = self.get_extra_headers(next_url, "GET", body) return self._do_request( diff --git a/rsconnect/main.py b/rsconnect/main.py index a0a2ffc0..15d5a824 100644 --- a/rsconnect/main.py +++ b/rsconnect/main.py @@ -316,7 +316,7 @@ def add(name, server, api_key, insecure, cacert, account, token, secret, verbose old_server = server_store.get_by_name(name) if account: - shinyapps_server = api.ShinyappsServer(server or "https://api.shinyapps.io", account, token, secret) + shinyapps_server = api.ShinyappsServer(server, account, token, secret) _test_shinyappsio_creds(shinyapps_server) server_store.set( diff --git a/tests/rsconnect-build-test/connect_remote_6443.json b/tests/rsconnect-build-test/connect_remote_6443.json new file mode 100644 index 00000000..32f7e05c --- /dev/null +++ b/tests/rsconnect-build-test/connect_remote_6443.json @@ -0,0 +1,83 @@ +{ + "rsconnect_build_running": false, + "rsconnect_content": { + "c96db3f3-87a1-4df5-9f58-eb109c397718": { + "guid": "c96db3f3-87a1-4df5-9f58-eb109c397718", + "bundle_id": "177", + "title": "orphan-proc-shiny-test", + "name": "orphan-proc-shiny-test", + "app_mode": "shiny", + "content_url": "https://connect.remote:6443/content/c96db3f3-87a1-4df5-9f58-eb109c397718/", + "dashboard_url": "https://connect.remote:6443/connect/#/apps/c96db3f3-87a1-4df5-9f58-eb109c397718", + "created_time": "2021-11-04T18:07:12Z", + "last_deployed_time": "2021-11-10T19:10:56Z", + "owner_guid": "edf26318-0027-4d9d-bbbb-54703ebb1855", + "rsconnect_build_status": "NEEDS_BUILD" + }, + "fe673896-f92a-40cc-be4c-e4872bb90a37": { + "guid": "fe673896-f92a-40cc-be4c-e4872bb90a37", + "bundle_id": "185", + "title": "interactive-rmd", + "name": "interactive-rmd", + "app_mode": "rmd-shiny", + "content_url": "https://connect.remote:6443/content/fe673896-f92a-40cc-be4c-e4872bb90a37/", + "dashboard_url": "https://connect.remote:6443/connect/#/apps/fe673896-f92a-40cc-be4c-e4872bb90a37", + "created_time": "2021-11-15T15:37:53Z", + "last_deployed_time": "2021-11-15T15:37:57Z", + "owner_guid": "edf26318-0027-4d9d-bbbb-54703ebb1855", + "rsconnect_build_status": "ERROR" + }, + "a0b6b5a2-5fbe-4293-8310-4f80054bc24f": { + "guid": "a0b6b5a2-5fbe-4293-8310-4f80054bc24f", + "bundle_id": "184", + "title": "stock-report-jupyter", + "name": "stock-report-jupyter", + "app_mode": "jupyter-static", + "content_url": "https://connect.remote:6443/content/a0b6b5a2-5fbe-4293-8310-4f80054bc24f/", + "dashboard_url": "https://connect.remote:6443/connect/#/apps/a0b6b5a2-5fbe-4293-8310-4f80054bc24f", + "created_time": "2021-11-15T15:27:18Z", + "last_deployed_time": "2021-11-15T15:35:27Z", + "owner_guid": "edf26318-0027-4d9d-bbbb-54703ebb1855", + "rsconnect_build_status": "RUNNING" + }, + "23315cc9-ed2a-40ad-9e99-e5e49066531a": { + "guid": "23315cc9-ed2a-40ad-9e99-e5e49066531a", + "bundle_id": "180", + "title": "static-rmd", + "name": "static-rmd2", + "app_mode": "rmd-static", + "content_url": "https://connect.remote:6443/content/23315cc9-ed2a-40ad-9e99-e5e49066531a/", + "dashboard_url": "https://connect.remote:6443/connect/#/apps/23315cc9-ed2a-40ad-9e99-e5e49066531a", + "created_time": "2021-11-15T15:20:58Z", + "last_deployed_time": "2021-11-15T15:25:31Z", + "owner_guid": "edf26318-0027-4d9d-bbbb-54703ebb1855", + "rsconnect_build_status": "COMPLETE", + "rsconnect_last_build_time": "2021-12-13T18:10:38Z", + "rsconnect_last_build_log": "/logs/localhost_3939/23315cc9-ed2a-40ad-9e99-e5e49066531a/ZUf44zVWHjODv1Rq.log", + "rsconnect_build_task_result": { + "id": "ZUf44zVWHjODv1Rq", + "user_id": 1, + "result": { + "type": "", + "data": null + }, + "finished": true, + "code": 0, + "error": "" + } + }, + "015143da-b75f-407c-81b1-99c4a724341e": { + "guid": "015143da-b75f-407c-81b1-99c4a724341e", + "bundle_id": "176", + "title": "plumber-async", + "name": "plumber-async", + "app_mode": "api", + "content_url": "https://connect.remote:6443/content/015143da-b75f-407c-81b1-99c4a724341e/", + "dashboard_url": "https://connect.remote:6443/connect/#/apps/015143da-b75f-407c-81b1-99c4a724341e", + "created_time": "2021-11-01T20:43:32Z", + "last_deployed_time": "2021-11-03T17:48:59Z", + "owner_guid": "edf26318-0027-4d9d-bbbb-54703ebb1855", + "rsconnect_build_status": "ERROR" + } + } +} \ No newline at end of file diff --git a/tests/test_main.py b/tests/test_main.py index ffb6cf19..0c8eccaf 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -1,3 +1,4 @@ +import json import os import shutil from os.path import join @@ -21,6 +22,14 @@ from rsconnect import VERSION +def _error_to_response(error): + """ + HTTPretty is unable to show errors resulting from callbacks, so this method attempts to raise failure visibility by + passing the return back through HTTP. + """ + return [500, {}, str(error)] + + class TestMain(TestCase): def setUp(self): shutil.rmtree("test-home", ignore_errors=True) @@ -95,6 +104,163 @@ def test_deploy_manifest(self): result = runner.invoke(cli, args) self.assertEqual(result.exit_code, 0, result.output) + # noinspection SpellCheckingInspection + @httpretty.activate(verbose=True, allow_net_connect=False) + def test_deploy_manifest_shinyapps(self): + original_api_key_value = os.environ.pop("CONNECT_API_KEY", None) + original_server_value = os.environ.pop("CONNECT_SERVER", None) + + httpretty.register_uri( + httpretty.GET, + "https://api.shinyapps.io/v1/users/me", + body=open("tests/testdata/shinyapps-responses/get-user.json", "r").read(), + status=200, + ) + httpretty.register_uri( + httpretty.GET, + "https://api.shinyapps.io/v1/applications?filter=name:like:shinyapp&offset=0&count=100&use_advanced_filters=true", + body=open("tests/testdata/shinyapps-responses/get-applications.json", "r").read(), + adding_headers={"Content-Type": "application/json"}, + status=200, + ) + httpretty.register_uri( + httpretty.GET, + "https://api.shinyapps.io/v1/accounts/", + body=open("tests/testdata/shinyapps-responses/get-accounts.json", "r").read(), + adding_headers={"Content-Type": "application/json"}, + status=200, + ) + + def post_application_callback(request, uri, response_headers): + parsed_request = json.loads(request.body) + try: + self.assertDictEqual(parsed_request, {"account": 82069, "name": "myapp", "template": "shiny"}) + except AssertionError as e: + return _error_to_response(e) + return [ + 201, + {"Content-Type": "application/json"}, + open("tests/testdata/shinyapps-responses/create-application.json", "r").read(), + ] + + httpretty.register_uri( + httpretty.POST, + "https://api.shinyapps.io/v1/applications/", + body=post_application_callback, + ) + + def post_bundle_callback(request, uri, response_headers): + parsed_request = json.loads(request.body) + del parsed_request["checksum"] + try: + self.assertDictEqual( + parsed_request, + { + "application": 8442, + "content_type": "application/x-tar", + "content_length": 10185, + }, + ) + except AssertionError as e: + return _error_to_response(e) + return [ + 201, + {"Content-Type": "application/json"}, + open("tests/testdata/shinyapps-responses/create-bundle.json", "r").read(), + ] + + httpretty.register_uri( + httpretty.POST, + "https://api.shinyapps.io/v1/bundles", + body=post_bundle_callback, + ) + + def s3_upload_callback(request, uri, response_headers): + try: + self.assertEqual(request.headers.get("content-length"), "10185") + self.assertEqual(len(request.body), 10185) + self.assertEqual(request.headers.get("content-md5"), "D1blMI4qTiI3tgeUOYXwkg==") + except AssertionError as e: + return _error_to_response(e) + return [201, {}, ""] + + httpretty.register_uri( + httpretty.PUT, + "https://lucid-uploads-staging.s3.amazonaws.com/bundles/application-8442/6c9ed0d91ee9426687d9ac231d47dc83.tar.gz?AWSAccessKeyId=theAccessKeyId&Signature=dGhlU2lnbmF0dXJlCg%3D%3D&content-md5=D1blMI4qTiI3tgeUOYXwkg%3D%3D&content-type=application%2Fx-tar&x-amz-security-token=dGhlVG9rZW4K&Expires=1656715153", + body=s3_upload_callback, + ) + + def post_bundle_status_callback(request, uri, response_headers): + parsed_request = json.loads(request.body) + try: + self.assertDictEqual(parsed_request, {"status": "ready"}) + except AssertionError as e: + return _error_to_response(e) + return [303, {"Location": "https://api.shinyapps.io/v1/bundles/12640"}, ""] + + httpretty.register_uri( + httpretty.POST, + "https://api.shinyapps.io/v1/bundles/12640/status", + body=post_bundle_status_callback, + ) + + httpretty.register_uri( + httpretty.GET, + "https://api.shinyapps.io/v1/bundles/12640", + body=open("tests/testdata/shinyapps-responses/get-accounts.json", "r").read(), + adding_headers={"Content-Type": "application/json"}, + status=200, + ) + + def post_deploy_callback(request, uri, response_headers): + parsed_request = json.loads(request.body) + try: + self.assertDictEqual(parsed_request, {"bundle": 12640, "rebuild": False}) + except AssertionError as e: + return _error_to_response(e) + return [ + 303, + {"Location": "https://api.shinyapps.io/v1/tasks/333"}, + open("tests/testdata/shinyapps-responses/post-deploy.json", "r").read(), + ] + + httpretty.register_uri( + httpretty.POST, + "https://api.shinyapps.io/v1/applications/8442/deploy", + body=post_deploy_callback, + ) + + httpretty.register_uri( + httpretty.GET, + "https://api.shinyapps.io/v1/tasks/333", + body=open("tests/testdata/shinyapps-responses/get-task.json", "r").read(), + adding_headers={"Content-Type": "application/json"}, + status=200, + ) + + runner = CliRunner() + args = [ + "deploy", + "manifest", + get_manifest_path("shinyapp"), + "--account", + "some-account", + "--token", + "someToken", + "--secret", + "c29tZVNlY3JldAo=", + "--title", + "myApp", + ] + try: + result = runner.invoke(cli, args) + self.assertEqual(result.exit_code, 0, result.output) + finally: + if original_api_key_value: + os.environ["CONNECT_API_KEY"] = original_api_key_value + if original_server_value: + os.environ["CONNECT_SERVER"] = original_server_value + def test_deploy_api(self): target = optional_target(get_api_path("flask")) runner = CliRunner() @@ -125,7 +291,7 @@ def test_add_shinyapps(self): [ "add", "--account", - "someAccount", + "some-account", "--name", "my-shinyapps", "--token", diff --git a/tests/test_metadata.py b/tests/test_metadata.py index ed15024c..74a2ec33 100644 --- a/tests/test_metadata.py +++ b/tests/test_metadata.py @@ -22,7 +22,7 @@ def setUp(self): self.server_store.set( "baz", "https://shinyapps.io", - account="someAccount", + account="some-account", token="someToken", secret="c29tZVNlY3JldAo=", ) @@ -60,7 +60,7 @@ def test_add(self): dict( name="baz", url="https://shinyapps.io", - account="someAccount", + account="some-account", token="someToken", secret="c29tZVNlY3JldAo=", ), diff --git a/tests/testdata/shinyapps-responses/create-application.json b/tests/testdata/shinyapps-responses/create-application.json new file mode 100644 index 00000000..f238305f --- /dev/null +++ b/tests/testdata/shinyapps-responses/create-application.json @@ -0,0 +1,124 @@ +{ + "id": 8442, + "name": "myapp", + "uuid": "799b44bc47c1424cab750fe9e5261792", + "url": "https://some-account.staging.shinyapps.io/myapp/", + "type": "shiny", + "mode": null, + "scheduler": "legacy", + "status": "pending", + "account_id": 1037, + "content_id": null, + "logplex_channel": null, + "logplex_token": null, + "storage_initialized": true, + "deployment_id": 10258, + "deployment": { + "id": 10258, + "application_memory_limit": 1024, + "application_cpu_limit": 1, + "application_timeout_minutes": 15, + "application_timeout_kill_minutes": 60, + "application_os_version": "xenial", + "image": { + "id": 3341, + "app_id": 8442, + "active": true, + "status": "ready", + "bundle_id": 485, + "bundle": { + "id": 485, + "app_id": 8442, + "user_id": 26, + "status": "ready", + "name": null, + "url": "s3://lucid-uploads-staging/bundles/application-8442/f34d4cd69fc74eeab7ddbf944b63c3af.tar.gz", + "checksum": "101f6fb239fdab6852f20257640abfda", + "parent_id": null, + "created_time": "2018-07-20T23:05:44", + "updated_time": "2018-07-20T23:05:45" + }, + "manifest": null, + "repository": "app-8442", + "registry": "registry01", + "tag": "image-3341", + "agents": [], + "created_time": "2018-07-20T23:05:47", + "updated_time": "2021-06-03T21:44:03" + }, + "properties": { + "application.visibility": "public", + "application.base-image.registry": null, + "application.base-image.repository": null, + "application.base-image.compiler": null, + "application.build-pool": null, + "application.ide.image.tag": null, + "application.jupyter.image.tag": null, + "application.initialize.image": null, + "application.initialize.image.tag": null, + "application.sidecar.image.tag": null, + "application.ide.autosave.on.idle": false, + "application.instances.template": "large", + "application.instances.start": 1, + "application.instances.max": 3, + "application.instances.load.factor": 0.5, + "application.instances.idle-threshold": 15, + "application.instances.down-threshold": 1, + "application.instances.fault-threshold": 1, + "application.instances.agent-pool": null, + "application.shiny.timeout.init": 60, + "application.shiny.timeout.idle": 5, + "application.shiny.timeout.conn": 900, + "application.shiny.timeout.read": 3600, + "application.shiny.timeout.session": 3600, + "application.shiny.timeout.reconnect": null, + "application.shiny.scheduler.max.requests": 20, + "application.shiny.scheduler.max.processes": 3, + "application.shiny.scheduler.min.processes": 0, + "application.shiny.scheduler.load.factor": 0.05, + "application.shiny.websockets": false, + "application.shiny.sockjs.protocols.disabled": null, + "application.connect.debug.log": "", + "application.connect.client.debugging": false, + "application.connect.client.transport.debugging": false, + "application.connect.version": "current", + "application.package.cache": true, + "application.connect.injection.version": 62, + "application.connect.branding": false, + "application.ide.version": "current", + "application.frontend.iFrameResizer.log": false, + "application.frontend.iFrameResizer.sizeHeight": true, + "application.frontend.iFrameResizer.sizeWidth": false, + "application.frontend.iFrameResizer.heightCalculationMethod": "bodyOffset", + "application.frontend.iFrameResizer.widthCalculationMethod": "bodyOffset", + "application.storage.size": "20G", + "application.unmigratable": "" + }, + "environment": {}, + "user": null, + "created_time": "2018-07-20T23:05:47", + "updated_time": "2018-07-20T23:05:47" + }, + "environment": {}, + "resources": { + "memory_limit": 1024, + "cpu_limit": 1, + "effective_memory_limit": 1024, + "effective_cpu_limit": 1 + }, + "configuration": { + "timeout_minutes": 15, + "timeout_kill_minutes": 60, + "effective_timeout_minutes": 15, + "effective_timeout_kill_minutes": 60 + }, + "runtime_options": null, + "next_deployment_id": null, + "prev_deployment_id": 10257, + "clone_parent_id": null, + "copy_parent_id": null, + "storage": [], + "exportable": true, + "created_time": "2018-07-20T22:46:41", + "updated_time": "2018-07-20T23:06:06" +} diff --git a/tests/testdata/shinyapps-responses/create-bundle.json b/tests/testdata/shinyapps-responses/create-bundle.json new file mode 100644 index 00000000..3d1d71e6 --- /dev/null +++ b/tests/testdata/shinyapps-responses/create-bundle.json @@ -0,0 +1,14 @@ +{ + "id": 12640, + "app_id": 8442, + "user_id": 47261, + "status": "pending", + "name": null, + "url": "s3://lucid-uploads-staging/bundles/application-1122350/6c9ed0d91ee9426687d9ac231d47dc83.tar.gz", + "checksum": "0f56e5308e2a4e2237b607943985f092", + "parent_id": null, + "created_time": "2022-07-01T21:39:13", + "updated_time": "2022-07-01T21:39:13", + "presigned_url": "https://lucid-uploads-staging.s3.amazonaws.com/bundles/application-8442/6c9ed0d91ee9426687d9ac231d47dc83.tar.gz?AWSAccessKeyId=theAccessKeyId&Signature=dGhlU2lnbmF0dXJlCg%3D%3D&content-md5=D1blMI4qTiI3tgeUOYXwkg%3D%3D&content-type=application%2Fx-tar&x-amz-security-token=dGhlVG9rZW4K&Expires=1656715153", + "presigned_checksum": "D1blMI4qTiI3tgeUOYXwkg==" +} diff --git a/tests/testdata/shinyapps-responses/get-accounts.json b/tests/testdata/shinyapps-responses/get-accounts.json new file mode 100644 index 00000000..1c31c457 --- /dev/null +++ b/tests/testdata/shinyapps-responses/get-accounts.json @@ -0,0 +1,198 @@ +{ + "count": 1, + "total": 1, + "offset": 0, + "accounts": [ + { + "id": 82069, + "name": "some-account", + "type": "individual", + "display_name": null, + "permissions": [ + "DELETE_LEGACY_TASK", + "CREATE_LEGACY_TASK_LOGS", + "MODIFY_VOLUME", + "DELETE_VOLUME", + "VIEW_ACCOUNT_TASK", + "VIEW_VOLUME", + "VIEW_DOMAIN", + "VIEW_ACCOUNT_NOTIFICATION_PREFERENCE", + "CREATE_APPLICATION", + "VIEW_ACCOUNT_USERS", + "CREATE_VOLUME", + "MANAGE_ACCOUNT_NOTIFICATION_PREFERENCE", + "DELETE_ACCOUNT", + "VIEW_ACCOUNT", + "DELETE_DOMAIN", + "VIEW_LEGACY_TASK", + "PURGE_VOLUME", + "MODIFY_LEGACY_TASK", + "CREATE_CONTENT_IN_NULL_SPACE", + "KILL_LEGACY_TASK", + "CREATE_SPACE", + "MANAGE_ACCOUNT", + "VIEW_ACCOUNT_USAGE", + "CREATE_CONTENT_IN_NAMED_SPACE", + "MANAGE_ACCOUNT_SUBSCRIPTION", + "VIEW_LEGACY_TASK_LOGS", + "PROVISION_VOLUME", + "MANAGE_ACCOUNT_ACCESS", + "CREATE_DOMAIN" + ], + "suspended_until": null, + "suspended_reason": null, + "suspended_pending": null, + "sso_enabled": false, + "sso_hint": null, + "beta_enabled": false, + "beta": { + "jupyter_enabled": false, + "publishing_enabled": true, + "sharing_enabled": false + }, + "owner": { + "id": 47261, + "first_name": "Example", + "last_name": "User", + "display_name": "Example User", + "organization": null, + "homepage": null, + "location": null, + "picture_url": "https://www.gravatar.com/avatar/e1bc03fb1393b9b1c3b3dc309cc0b4cf", + "email": "example.user@rstudio.com", + "superuser": false, + "email_verified": true, + "local_auth": true, + "referral": null, + "google_auth_id": "100000000000000000000", + "github_auth_id": null, + "github_auth_token": null, + "last_login_attempt": "2022-05-31T22:20:28", + "login_attempts": 0, + "lockout_until": null, + "sso_account_id": null, + "grant": null, + "created_time": "2021-09-28T19:32:47", + "updated_time": "2022-05-31T22:20:28" + }, + "subscription": { + "id": 82069, + "type": { + "id": 4, + "name": "professional", + "premium": true, + "created_time": "2016-11-03T15:37:09", + "updated_time": "2016-11-03T15:37:09" + }, + "start_date": null, + "end_date": null, + "cancel_date": null, + "cycle_start_date": "2022-06-28T19:32:52", + "cycle_end_date": "2022-07-28T19:32:52", + "status": null, + "plan": null, + "discount": null, + "billing": { + "name": null, + "email": null, + "organization": null, + "address": { + "address_1": null, + "address_2": null, + "city": null, + "state": null, + "zip": null, + "country": null + } + }, + "payment": null, + "entitlements": { + "AuthenticationFeature": { + "enabled": true + }, + "BrandingFeature": { + "enabled": true + }, + "MaxApplicationsLimit": { + "limit": 1000 + }, + "MaxInstancesLimit": { + "limit": 10 + }, + "InstanceIdleLimit": { + "limit": 480 + }, + "BundleSizeLimit": { + "limit": 50000000000 + }, + "MaxCustomDomainsLimit": { + "limit": 100 + }, + "MaxAccountUsersLimit": { + "limit": 25 + }, + "MaxApplicationWorkersLimit": { + "limit": 10 + }, + "MaxApplicationHoursLimit": { + "limit": 10000 + }, + "MaxProjectsLimit": { + "limit": 100000 + }, + "MaxPublicSpacesLimit": { + "limit": 100 + }, + "MaxPrivateSpacesLimit": { + "limit": 100 + }, + "MaxApplicationCpuLimit": { + "limit": 1 + }, + "MaxApplicationMemoryLimit": { + "limit": 1024 + }, + "ApplicationInstanceType": { + "items": [ + "small", + "medium", + "large", + "xlarge", + "xxlarge", + "xxxlarge" + ] + } + }, + "created_time": "2021-09-28T19:32:52", + "updated_time": "2021-09-28T19:32:52" + }, + "licenses": [ + { + "id": 103876, + "type": "cloud", + "status": "active", + "name": "cloud-premium", + "expires": null, + "exempt": false, + "exempt_until": null, + "suspended": false, + "suspended_until": null, + "suspended_reason": null, + "suspension_pending": false, + "suspension_pending_until": null, + "cycle_anchor": "2021-11-09T16:09:31", + "cycle_start": "2022-06-09T16:09:31", + "cycle_end": "2022-07-09T16:09:31", + "overage_until": null, + "overage": false, + "created_time": "2021-09-28T19:32:52", + "updated_time": "2021-12-06T21:11:49" + } + ], + "account_role": "none", + "stripe_customer_id": null, + "created_time": "2021-09-28T19:32:52", + "updated_time": "2021-09-28T19:32:52" + } + ] +} diff --git a/tests/testdata/shinyapps-responses/get-applications.json b/tests/testdata/shinyapps-responses/get-applications.json new file mode 100644 index 00000000..db604309 --- /dev/null +++ b/tests/testdata/shinyapps-responses/get-applications.json @@ -0,0 +1,6 @@ +{ + "count": 0, + "total": 0, + "offset": 0, + "applications": [] +} diff --git a/tests/testdata/shinyapps-responses/get-bundle.json b/tests/testdata/shinyapps-responses/get-bundle.json new file mode 100644 index 00000000..20c42f4e --- /dev/null +++ b/tests/testdata/shinyapps-responses/get-bundle.json @@ -0,0 +1,12 @@ +{ + "id": 12640, + "app_id": 8442, + "user_id": 47261, + "status": "ready", + "name": null, + "url": "s3://lucid-uploads-staging/bundles/application-8442/6c9ed0d91ee9426687d9ac231d47dc83.tar.gz", + "checksum": "0f56e5308e2a4e2237b607943985f092", + "parent_id": null, + "created_time": "2022-07-01T21:39:13", + "updated_time": "2022-07-01T21:39:14" +} diff --git a/tests/testdata/shinyapps-responses/get-task.json b/tests/testdata/shinyapps-responses/get-task.json new file mode 100644 index 00000000..0bdb2962 --- /dev/null +++ b/tests/testdata/shinyapps-responses/get-task.json @@ -0,0 +1,17 @@ +{ + "id": 333, + "account_id": 82069, + "parent_id": null, + "action": "application-deploy", + "description": "Stopping old instances", + "status": "success", + "error": null, + "finished": true, + "finished_time": "2022-07-05T16:07:16", + "arguments": { + "app_id": 8442, + "deployment_id": 850618 + }, + "created_time": "2022-07-05T16:06:42", + "updated_time": "2022-07-05T16:07:16" +} diff --git a/tests/testdata/shinyapps-responses/get-user.json b/tests/testdata/shinyapps-responses/get-user.json new file mode 100644 index 00000000..1e5351fb --- /dev/null +++ b/tests/testdata/shinyapps-responses/get-user.json @@ -0,0 +1,25 @@ +{ + "id": 47261, + "first_name": "Example", + "last_name": "User", + "display_name": "Example User", + "organization": null, + "homepage": null, + "location": null, + "picture_url": "https://www.gravatar.com/avatar/e1bc03fb1393b9b1c3b3dc309cc0b4cf", + "email": "example.user@rstudio.com", + "superuser": false, + "email_verified": true, + "local_auth": true, + "referral": null, + "google_auth_id": "100000000000000000000", + "github_auth_id": null, + "github_auth_token": null, + "last_login_attempt": "2022-05-31T22:20:28", + "login_attempts": 0, + "lockout_until": null, + "sso_account_id": null, + "grant": null, + "created_time": "2021-09-28T19:32:47", + "updated_time": "2022-05-31T22:20:28" +} diff --git a/tests/testdata/shinyapps-responses/post-deploy.json b/tests/testdata/shinyapps-responses/post-deploy.json new file mode 100644 index 00000000..c045572f --- /dev/null +++ b/tests/testdata/shinyapps-responses/post-deploy.json @@ -0,0 +1,3 @@ +{ + "task_id": 333 +} From c0f425be693230076a1b233dbe0aab8b796b892c Mon Sep 17 00:00:00 2001 From: Matthew Lynch Date: Tue, 5 Jul 2022 11:21:15 -0500 Subject: [PATCH 40/57] fix formatting --- tests/test_main.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/test_main.py b/tests/test_main.py index 0c8eccaf..4963ec23 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -186,7 +186,14 @@ def s3_upload_callback(request, uri, response_headers): httpretty.register_uri( httpretty.PUT, - "https://lucid-uploads-staging.s3.amazonaws.com/bundles/application-8442/6c9ed0d91ee9426687d9ac231d47dc83.tar.gz?AWSAccessKeyId=theAccessKeyId&Signature=dGhlU2lnbmF0dXJlCg%3D%3D&content-md5=D1blMI4qTiI3tgeUOYXwkg%3D%3D&content-type=application%2Fx-tar&x-amz-security-token=dGhlVG9rZW4K&Expires=1656715153", + "https://lucid-uploads-staging.s3.amazonaws.com/bundles/application-8442/" + "6c9ed0d91ee9426687d9ac231d47dc83.tar.gz" + "?AWSAccessKeyId=theAccessKeyId" + "&Signature=dGhlU2lnbmF0dXJlCg%3D%3D" + "&content-md5=D1blMI4qTiI3tgeUOYXwkg%3D%3D" + "&content-type=application%2Fx-tar" + "&x-amz-security-token=dGhlVG9rZW4K" + "&Expires=1656715153", body=s3_upload_callback, ) From dc89ec677bf817b265695d7eed0bbe82d65d1bac Mon Sep 17 00:00:00 2001 From: Matthew Lynch Date: Tue, 5 Jul 2022 11:22:47 -0500 Subject: [PATCH 41/57] fix formatting --- tests/test_main.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_main.py b/tests/test_main.py index 4963ec23..1fe9c262 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -118,7 +118,8 @@ def test_deploy_manifest_shinyapps(self): ) httpretty.register_uri( httpretty.GET, - "https://api.shinyapps.io/v1/applications?filter=name:like:shinyapp&offset=0&count=100&use_advanced_filters=true", + "https://api.shinyapps.io/v1/applications" + "?filter=name:like:shinyapp&offset=0&count=100&use_advanced_filters=true", body=open("tests/testdata/shinyapps-responses/get-applications.json", "r").read(), adding_headers={"Content-Type": "application/json"}, status=200, From feb886fb11eb23fddbab87b222eb08773a60af2c Mon Sep 17 00:00:00 2001 From: Matthew Lynch Date: Tue, 5 Jul 2022 11:29:34 -0500 Subject: [PATCH 42/57] show response body for http call failures --- rsconnect/api.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/rsconnect/api.py b/rsconnect/api.py index 27b7ba07..5cbe650a 100644 --- a/rsconnect/api.py +++ b/rsconnect/api.py @@ -52,8 +52,14 @@ def handle_bad_response(self, response): raise RSConnectException(error) if response.status < 200 or response.status > 299: raise RSConnectException( - "Received an unexpected response from %s (calling %s): %s %s" - % (self.remote_name, response.full_uri, response.status, response.reason) + "Received an unexpected response from %s (calling %s): %s %s\n%s" + % ( + self.remote_name, + response.full_uri, + response.status, + response.reason, + response.response_body, + ) ) From db8b08cc1315f5cb820fa2db9fd693ee367392c4 Mon Sep 17 00:00:00 2001 From: Matthew Lynch Date: Tue, 5 Jul 2022 11:37:48 -0500 Subject: [PATCH 43/57] remove tar.gz assertions against properties that vary by platform --- tests/test_main.py | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/tests/test_main.py b/tests/test_main.py index 1fe9c262..89258556 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -148,18 +148,19 @@ def post_application_callback(request, uri, response_headers): httpretty.POST, "https://api.shinyapps.io/v1/applications/", body=post_application_callback, + status=200, ) def post_bundle_callback(request, uri, response_headers): parsed_request = json.loads(request.body) del parsed_request["checksum"] + del parsed_request["content_length"] try: self.assertDictEqual( parsed_request, { "application": 8442, "content_type": "application/x-tar", - "content_length": 10185, }, ) except AssertionError as e: @@ -176,15 +177,6 @@ def post_bundle_callback(request, uri, response_headers): body=post_bundle_callback, ) - def s3_upload_callback(request, uri, response_headers): - try: - self.assertEqual(request.headers.get("content-length"), "10185") - self.assertEqual(len(request.body), 10185) - self.assertEqual(request.headers.get("content-md5"), "D1blMI4qTiI3tgeUOYXwkg==") - except AssertionError as e: - return _error_to_response(e) - return [201, {}, ""] - httpretty.register_uri( httpretty.PUT, "https://lucid-uploads-staging.s3.amazonaws.com/bundles/application-8442/" @@ -195,7 +187,7 @@ def s3_upload_callback(request, uri, response_headers): "&content-type=application%2Fx-tar" "&x-amz-security-token=dGhlVG9rZW4K" "&Expires=1656715153", - body=s3_upload_callback, + body="", ) def post_bundle_status_callback(request, uri, response_headers): From 8a0d95ff240d6524a914a1349179cab46da8012a Mon Sep 17 00:00:00 2001 From: Matthew Lynch Date: Tue, 5 Jul 2022 11:51:36 -0500 Subject: [PATCH 44/57] tweak test json loading for ci --- tests/test_main.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/test_main.py b/tests/test_main.py index 89258556..6860daf1 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -30,6 +30,12 @@ def _error_to_response(error): return [500, {}, str(error)] +def load_json(data): + if isinstance(data, bytes): + return json.loads(data.decode()) + return json.loads(data) + + class TestMain(TestCase): def setUp(self): shutil.rmtree("test-home", ignore_errors=True) From 27d3836d34f5d48670c8dc95b6697e24a353d1ea Mon Sep 17 00:00:00 2001 From: Matthew Lynch Date: Tue, 5 Jul 2022 11:57:29 -0500 Subject: [PATCH 45/57] tweak test json loading for ci --- tests/test_main.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_main.py b/tests/test_main.py index 6860daf1..c7e390f8 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -139,7 +139,7 @@ def test_deploy_manifest_shinyapps(self): ) def post_application_callback(request, uri, response_headers): - parsed_request = json.loads(request.body) + parsed_request = load_json(request.body) try: self.assertDictEqual(parsed_request, {"account": 82069, "name": "myapp", "template": "shiny"}) except AssertionError as e: @@ -158,7 +158,7 @@ def post_application_callback(request, uri, response_headers): ) def post_bundle_callback(request, uri, response_headers): - parsed_request = json.loads(request.body) + parsed_request = load_json(request.body) del parsed_request["checksum"] del parsed_request["content_length"] try: @@ -197,7 +197,7 @@ def post_bundle_callback(request, uri, response_headers): ) def post_bundle_status_callback(request, uri, response_headers): - parsed_request = json.loads(request.body) + parsed_request = load_json(request.body) try: self.assertDictEqual(parsed_request, {"status": "ready"}) except AssertionError as e: @@ -219,7 +219,7 @@ def post_bundle_status_callback(request, uri, response_headers): ) def post_deploy_callback(request, uri, response_headers): - parsed_request = json.loads(request.body) + parsed_request = load_json(request.body) try: self.assertDictEqual(parsed_request, {"bundle": 12640, "rebuild": False}) except AssertionError as e: From 78e0877e1c3b60d7e9f510fe206f10abb039e3dd Mon Sep 17 00:00:00 2001 From: Matthew Lynch Date: Tue, 5 Jul 2022 12:05:09 -0500 Subject: [PATCH 46/57] initialize S3 client like other clients Initialize S3 server like other servers --- rsconnect/api.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/rsconnect/api.py b/rsconnect/api.py index 5cbe650a..fc08ca8d 100644 --- a/rsconnect/api.py +++ b/rsconnect/api.py @@ -95,10 +95,8 @@ def __init__(self, url, api_key, insecure=False, ca_data=None): class S3Server(AbstractRemoteServer): - remote_name = "S3" - def __init__(self, url: str): - self.url = url + super().__init__(url, "S3") class RSConnectClient(HTTPServer): From 1a9d1a55d0bd65e08e0713cd544e8debc4cd450a Mon Sep 17 00:00:00 2001 From: Matthew Lynch Date: Tue, 5 Jul 2022 12:24:33 -0500 Subject: [PATCH 47/57] log task id and allow more time for task completion --- rsconnect/api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rsconnect/api.py b/rsconnect/api.py index fc08ca8d..4495aae8 100644 --- a/rsconnect/api.py +++ b/rsconnect/api.py @@ -989,7 +989,7 @@ def get_task(self, task_id): def get_current_user(self): return self.get("/v1/users/me") - def wait_until_task_is_successful(self, task_id, timeout=60): + def wait_until_task_is_successful(self, task_id, timeout=180): counter = 1 status = None @@ -999,7 +999,7 @@ def wait_until_task_is_successful(self, task_id, timeout=60): status = task.json_data["status"] description = task.json_data["description"] - print("Waiting: {} - {}".format(status, description)) + print("Waiting on task {}: {} - {}".format(task_id, status, description)) if status == "success": break From bea0a029fbbe5fae42ea6902f4fa7afe4c965565 Mon Sep 17 00:00:00 2001 From: Matthew Lynch Date: Tue, 5 Jul 2022 16:09:16 -0500 Subject: [PATCH 48/57] rename account to account_name --- rsconnect/api.py | 10 +++++----- rsconnect/main.py | 2 +- rsconnect/metadata.py | 12 ++++++------ rsconnect/validation.py | 4 ++-- tests/test_metadata.py | 4 ++-- 5 files changed, 16 insertions(+), 16 deletions(-) diff --git a/rsconnect/api.py b/rsconnect/api.py index 4495aae8..00f8a8a4 100644 --- a/rsconnect/api.py +++ b/rsconnect/api.py @@ -342,7 +342,7 @@ def __init__( insecure=insecure, cacert=cacert, ca_data=ca_data, - account=account, + account_name=account, token=token, secret=secret, ) @@ -380,7 +380,7 @@ def setup_remote_server( insecure: bool = False, cacert: IO = None, ca_data: str = None, - account: str = None, + account_name: str = None, token: str = None, secret: str = None, ): @@ -390,7 +390,7 @@ def setup_remote_server( api_key=api_key, insecure=insecure, cacert=cacert, - account=account, + account_name=account_name, token=token, secret=secret, ) @@ -404,7 +404,7 @@ def setup_remote_server( api_key = server_data.api_key insecure = server_data.insecure ca_data = server_data.ca_data - account = server_data.account + account_name = server_data.account_name token = server_data.token secret = server_data.secret self.is_server_from_store = server_data.from_store @@ -412,7 +412,7 @@ def setup_remote_server( if api_key: self.remote_server = RSConnectServer(url, api_key, insecure, ca_data) else: - self.remote_server = ShinyappsServer(url, account, token, secret) + self.remote_server = ShinyappsServer(url, account_name, token, secret) def setup_client(self, cookies=None, timeout=30, **kwargs): if isinstance(self.remote_server, RSConnectServer): diff --git a/rsconnect/main.py b/rsconnect/main.py index 15d5a824..c1777021 100644 --- a/rsconnect/main.py +++ b/rsconnect/main.py @@ -322,7 +322,7 @@ def add(name, server, api_key, insecure, cacert, account, token, secret, verbose server_store.set( name, shinyapps_server.url, - account=shinyapps_server.account_name, + account_name=shinyapps_server.account_name, token=shinyapps_server.token, secret=shinyapps_server.secret, ) diff --git a/rsconnect/metadata.py b/rsconnect/metadata.py index 2f023bfc..33fa9d99 100644 --- a/rsconnect/metadata.py +++ b/rsconnect/metadata.py @@ -227,7 +227,7 @@ def __init__( api_key: typing.Optional[str] = None, insecure: typing.Optional[bool] = None, ca_data: typing.Optional[str] = None, - account: typing.Optional[str] = None, + account_name: typing.Optional[str] = None, token: typing.Optional[str] = None, secret: typing.Optional[str] = None, ): @@ -237,7 +237,7 @@ def __init__( self.api_key = api_key self.insecure = insecure self.ca_data = ca_data - self.account = account + self.account_name = account_name self.token = token self.secret = secret @@ -277,7 +277,7 @@ def get_all_servers(self): """ return self._get_sorted_values(lambda s: s["name"]) - def set(self, name, url, api_key=None, insecure=False, ca_data=None, account=None, token=None, secret=None): + def set(self, name, url, api_key=None, insecure=False, ca_data=None, account_name=None, token=None, secret=None): """ Add (or update) information about a Connect server @@ -286,7 +286,7 @@ def set(self, name, url, api_key=None, insecure=False, ca_data=None, account=Non :param api_key: the API key to use to authenticate with the Connect server. :param insecure: a flag to disable TLS verification. :param ca_data: client side certificate data to use for TLS. - :param account: shinyapps.io account name. + :param account_name: shinyapps.io account name. :param token: shinyapps.io token. :param secret: shinyapps.io secret. """ @@ -297,7 +297,7 @@ def set(self, name, url, api_key=None, insecure=False, ca_data=None, account=Non if api_key: target_data = dict(api_key=api_key, insecure=insecure, ca_cert=ca_data) else: - target_data = dict(account=account, token=token, secret=secret) + target_data = dict(account_name=account_name, token=token, secret=secret) self._set(name, {**common_data, **target_data}) def remove_by_name(self, name): @@ -358,7 +358,7 @@ def resolve(self, name, url): insecure=entry.get("insecure"), ca_data=entry.get("ca_cert"), api_key=entry.get("api_key"), - account=entry.get("account"), + account_name=entry.get("account_name"), token=entry.get("token"), secret=entry.get("secret"), ) diff --git a/rsconnect/validation.py b/rsconnect/validation.py index 747d1113..718be824 100644 --- a/rsconnect/validation.py +++ b/rsconnect/validation.py @@ -11,13 +11,13 @@ def validate_new_server_options(url, api_key, insecure, cacert, account, token, validate_connection_options(None, url, api_key, insecure, cacert, account, token, secret) -def validate_connection_options(name, url, api_key, insecure, cacert, account, token, secret): +def validate_connection_options(name, url, api_key, insecure, cacert, account_name, token, secret): """ Validates provided Connect or shinyapps.io connection options and returns which target to use given the provided options. """ connect_options = {"-k/--api-key": api_key, "-i/--insecure": insecure, "-c/--cacert": cacert} - shinyapps_options = {"-T/--token": token, "-S/--secret": secret, "-a/--account": account} + shinyapps_options = {"-T/--token": token, "-S/--secret": secret, "-a/--account": account_name} options_mutually_exclusive_with_name = {"-s/--server": url, **connect_options, **shinyapps_options} present_options_mutually_exclusive_with_name = _get_present_options(options_mutually_exclusive_with_name) diff --git a/tests/test_metadata.py b/tests/test_metadata.py index 74a2ec33..651d20cf 100644 --- a/tests/test_metadata.py +++ b/tests/test_metadata.py @@ -22,7 +22,7 @@ def setUp(self): self.server_store.set( "baz", "https://shinyapps.io", - account="some-account", + account_name="some-account", token="someToken", secret="c29tZVNlY3JldAo=", ) @@ -60,7 +60,7 @@ def test_add(self): dict( name="baz", url="https://shinyapps.io", - account="some-account", + account_name="some-account", token="someToken", secret="c29tZVNlY3JldAo=", ), From a257f84bea48f1274a2dadaa50929e2ec6ea400d Mon Sep 17 00:00:00 2001 From: Matthew Lynch Date: Tue, 5 Jul 2022 16:10:03 -0500 Subject: [PATCH 49/57] better remote target inference --- rsconnect/api.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/rsconnect/api.py b/rsconnect/api.py index 00f8a8a4..189beb22 100644 --- a/rsconnect/api.py +++ b/rsconnect/api.py @@ -411,8 +411,10 @@ def setup_remote_server( if api_key: self.remote_server = RSConnectServer(url, api_key, insecure, ca_data) - else: + elif token and secret: self.remote_server = ShinyappsServer(url, account_name, token, secret) + else: + raise RSConnectException("Unable to infer Connect server type and setup server.") def setup_client(self, cookies=None, timeout=30, **kwargs): if isinstance(self.remote_server, RSConnectServer): From 25d0d17a65345468cb147856b0ac9567358ca4ff Mon Sep 17 00:00:00 2001 From: Matthew Lynch Date: Tue, 5 Jul 2022 16:19:20 -0500 Subject: [PATCH 50/57] import RSConnectException from exception instead of api --- rsconnect/actions.py | 4 ++-- rsconnect/validation.py | 12 +++++------- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/rsconnect/actions.py b/rsconnect/actions.py index 8fc078a9..6fbaecc0 100644 --- a/rsconnect/actions.py +++ b/rsconnect/actions.py @@ -223,8 +223,8 @@ def test_shinyapps_server(server: api.ShinyappsServer): try: result = client.get_current_user() server.handle_bad_response(result) - except api.RSConnectException as exc: - raise api.RSConnectException("Failed to verify with shinyapps.io ({}).".format(exc)) + except RSConnectException as exc: + raise RSConnectException("Failed to verify with shinyapps.io ({}).".format(exc)) def test_api_key(connect_server): diff --git a/rsconnect/validation.py b/rsconnect/validation.py index 718be824..d8cdb999 100644 --- a/rsconnect/validation.py +++ b/rsconnect/validation.py @@ -1,6 +1,6 @@ import typing -from . import api +from rsconnect.exception import RSConnectException def _get_present_options(options: typing.Dict[str, typing.Optional[str]]) -> typing.List[str]: @@ -22,13 +22,13 @@ def validate_connection_options(name, url, api_key, insecure, cacert, account_na present_options_mutually_exclusive_with_name = _get_present_options(options_mutually_exclusive_with_name) if name and present_options_mutually_exclusive_with_name: - raise api.RSConnectException( + raise RSConnectException( "-n/--name cannot be specified in conjunction with options {}".format( ", ".join(present_options_mutually_exclusive_with_name) ) ) if not name and not url and not shinyapps_options: - raise api.RSConnectException( + raise RSConnectException( "You must specify one of -n/--name OR -s/--server OR -a/--account, -T/--token, -S/--secret." ) @@ -36,7 +36,7 @@ def validate_connection_options(name, url, api_key, insecure, cacert, account_na present_shinyapps_options = _get_present_options(shinyapps_options) if present_connect_options and present_shinyapps_options: - raise api.RSConnectException( + raise RSConnectException( "Connect options ({}) may not be passed alongside shinyapps.io options ({}).".format( ", ".join(present_connect_options), ", ".join(present_shinyapps_options) ) @@ -44,6 +44,4 @@ def validate_connection_options(name, url, api_key, insecure, cacert, account_na if present_shinyapps_options: if len(present_shinyapps_options) != 3: - raise api.RSConnectException( - "-a/--account, -T/--token, and -S/--secret must all be provided for shinyapps.io." - ) + raise RSConnectException("-a/--account, -T/--token, and -S/--secret must all be provided for shinyapps.io.") From f502e446a1af03130f6f2281ae0abf1c73513192 Mon Sep 17 00:00:00 2001 From: Matthew Lynch Date: Tue, 5 Jul 2022 16:29:42 -0500 Subject: [PATCH 51/57] more exhaustive type checks for connect/shinyapps --- rsconnect/actions.py | 4 +++- rsconnect/api.py | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/rsconnect/actions.py b/rsconnect/actions.py index 6fbaecc0..55fdbc94 100644 --- a/rsconnect/actions.py +++ b/rsconnect/actions.py @@ -1531,9 +1531,11 @@ def _gather_basic_deployment_info_for_framework( if isinstance(remote_server, api.RSConnectServer): app = api.get_app_info(remote_server, app_id) existing_app_mode = AppModes.get_by_ordinal(app.get("app_mode", 0), True) - else: + elif isinstance(remote_server, api.ShinyappsServer): app = api.get_shinyapp_info(remote_server, app_id) existing_app_mode = AppModes.get_by_cloud_name(app.json_data["mode"]) + else: + raise RSConnectException("Unable to infer Connect client.") if existing_app_mode and app_mode != existing_app_mode: msg = ( "Deploying with mode '%s',\n" diff --git a/rsconnect/api.py b/rsconnect/api.py index 189beb22..e2f3d2e5 100644 --- a/rsconnect/api.py +++ b/rsconnect/api.py @@ -765,9 +765,11 @@ def validate_app_mode(self, *args, **kwargs): if isinstance(self.remote_server, RSConnectServer): app = get_app_info(self.remote_server, app_id) existing_app_mode = AppModes.get_by_ordinal(app.get("app_mode", 0), True) - else: + elif isinstance(self.remote_server, ShinyappsServer): app = get_shinyapp_info(self.remote_server, app_id) existing_app_mode = AppModes.get_by_cloud_name(app.json_data["mode"]) + else: + raise RSConnectException("Unable to infer Connect client.") if existing_app_mode and app_mode != existing_app_mode: msg = ( "Deploying with mode '%s',\n" From 8b18d781de51d01e82baf12247a5bba4ab18d6cc Mon Sep 17 00:00:00 2001 From: Matthew Lynch Date: Wed, 6 Jul 2022 11:20:20 -0500 Subject: [PATCH 52/57] error handling for shinyapps.io task failures --- rsconnect/api.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/rsconnect/api.py b/rsconnect/api.py index e2f3d2e5..95dda0ba 100644 --- a/rsconnect/api.py +++ b/rsconnect/api.py @@ -913,6 +913,9 @@ def __init__(self, app_id: int, app_url: str, bundle_id: int, presigned_url: str class ShinyappsClient(HTTPServer): + + _TERMINAL_STATUSES = {"success", "failed", "error"} + def __init__(self, shinyapps_server: ShinyappsServer, timeout: int = 30): self._token = shinyapps_server.token self._key = base64.b64decode(shinyapps_server.secret) @@ -978,7 +981,6 @@ def create_bundle(self, application_id: int, content_type: str, content_length: "checksum": checksum, } result = self.post("/v1/bundles", body=bundle_data) - print(result.json_data) return result def set_bundle_status(self, bundle_id, bundle_status): @@ -994,22 +996,24 @@ def get_current_user(self): return self.get("/v1/users/me") def wait_until_task_is_successful(self, task_id, timeout=180): - counter = 1 - status = None - - while counter < timeout and status not in ["success", "failed", "error"]: + start_time = time.time() + while time.time() - start_time < timeout: task = self.get_task(task_id) self._server.handle_bad_response(task) status = task.json_data["status"] description = task.json_data["description"] - - print("Waiting on task {}: {} - {}".format(task_id, status, description)) + error = task.json_data["error"] if status == "success": break + if status in {"failed", "error"}: + raise RSConnectException("Task {} {} - {} (error: {})".format(task_id, status, description, error)) + + print("Waiting on task {}: {} - {}".format(task_id, status, description)) time.sleep(2) - counter += 1 + + print("Task done: {}".format(description)) def prepare_deploy(self, app_id: typing.Optional[str], app_name: str, bundle_size: int, bundle_hash: str): From 0c3146b4b7a769783d0c2bbe468186972bbefe91 Mon Sep 17 00:00:00 2001 From: Matthew Lynch Date: Wed, 6 Jul 2022 12:08:25 -0500 Subject: [PATCH 53/57] make shinyapps output consistent with rsconnect --- rsconnect/api.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/rsconnect/api.py b/rsconnect/api.py index 95dda0ba..2a77221c 100644 --- a/rsconnect/api.py +++ b/rsconnect/api.py @@ -658,6 +658,7 @@ def deploy_bundle( self.client.do_deploy(prepare_deploy_result.bundle_id, prepare_deploy_result.app_id) + print("Application successfully deployed to {}".format(prepare_deploy_result.app_url)) webbrowser.open_new(prepare_deploy_result.app_url) self.state["deployed_info"] = { @@ -996,6 +997,8 @@ def get_current_user(self): return self.get("/v1/users/me") def wait_until_task_is_successful(self, task_id, timeout=180): + print() + print("Waiting for task: {}".format(task_id)) start_time = time.time() while time.time() - start_time < timeout: task = self.get_task(task_id) @@ -1008,9 +1011,9 @@ def wait_until_task_is_successful(self, task_id, timeout=180): break if status in {"failed", "error"}: - raise RSConnectException("Task {} {} - {} (error: {})".format(task_id, status, description, error)) + raise RSConnectException("Application deployment failed with error: {}".format(error)) - print("Waiting on task {}: {} - {}".format(task_id, status, description)) + print(" {} - {}".format(status, description)) time.sleep(2) From 4b1a9412545c8b69ab345dd278fed62171bca156 Mon Sep 17 00:00:00 2001 From: Matthew Lynch Date: Wed, 6 Jul 2022 13:40:16 -0500 Subject: [PATCH 54/57] rename load_json -> _load_json --- tests/test_main.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/test_main.py b/tests/test_main.py index c7e390f8..f2c39dcc 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -30,7 +30,7 @@ def _error_to_response(error): return [500, {}, str(error)] -def load_json(data): +def _load_json(data): if isinstance(data, bytes): return json.loads(data.decode()) return json.loads(data) @@ -139,7 +139,7 @@ def test_deploy_manifest_shinyapps(self): ) def post_application_callback(request, uri, response_headers): - parsed_request = load_json(request.body) + parsed_request = _load_json(request.body) try: self.assertDictEqual(parsed_request, {"account": 82069, "name": "myapp", "template": "shiny"}) except AssertionError as e: @@ -158,7 +158,7 @@ def post_application_callback(request, uri, response_headers): ) def post_bundle_callback(request, uri, response_headers): - parsed_request = load_json(request.body) + parsed_request = _load_json(request.body) del parsed_request["checksum"] del parsed_request["content_length"] try: @@ -197,7 +197,7 @@ def post_bundle_callback(request, uri, response_headers): ) def post_bundle_status_callback(request, uri, response_headers): - parsed_request = load_json(request.body) + parsed_request = _load_json(request.body) try: self.assertDictEqual(parsed_request, {"status": "ready"}) except AssertionError as e: @@ -219,7 +219,7 @@ def post_bundle_status_callback(request, uri, response_headers): ) def post_deploy_callback(request, uri, response_headers): - parsed_request = load_json(request.body) + parsed_request = _load_json(request.body) try: self.assertDictEqual(parsed_request, {"bundle": 12640, "rebuild": False}) except AssertionError as e: From 8dbce3bade9c8475ab5878fbc39aa4485471a65e Mon Sep 17 00:00:00 2001 From: Matthew Lynch Date: Wed, 6 Jul 2022 13:40:46 -0500 Subject: [PATCH 55/57] formatting --- rsconnect/api.py | 1 - 1 file changed, 1 deletion(-) diff --git a/rsconnect/api.py b/rsconnect/api.py index 2a77221c..33d774e1 100644 --- a/rsconnect/api.py +++ b/rsconnect/api.py @@ -1016,7 +1016,6 @@ def wait_until_task_is_successful(self, task_id, timeout=180): print(" {} - {}".format(status, description)) time.sleep(2) - print("Task done: {}".format(description)) def prepare_deploy(self, app_id: typing.Optional[str], app_name: str, bundle_size: int, bundle_hash: str): From 6dc0a5519442547c3ebe866265c016de339e2a8c Mon Sep 17 00:00:00 2001 From: Matthew Lynch Date: Fri, 8 Jul 2022 14:05:19 -0500 Subject: [PATCH 56/57] remove redundant click decorators --- rsconnect/main.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/rsconnect/main.py b/rsconnect/main.py index c1777021..76fd7ffb 100644 --- a/rsconnect/main.py +++ b/rsconnect/main.py @@ -734,8 +734,6 @@ def deploy_notebook( envvar="SHINYAPPS_ACCOUNT", help="The shinyapps.io account name.", ) -@server_args -@content_args @click.option( "--token", "-T", From 33c0c18b7b4a5929dc9fa8c0fa19e4f7949eb7e6 Mon Sep 17 00:00:00 2001 From: Matthew Lynch Date: Fri, 8 Jul 2022 14:10:18 -0500 Subject: [PATCH 57/57] remove validate_new_server_options in favor of validate_connection_options --- rsconnect/api.py | 2 +- rsconnect/main.py | 4 ++-- rsconnect/validation.py | 6 +----- 3 files changed, 4 insertions(+), 8 deletions(-) diff --git a/rsconnect/api.py b/rsconnect/api.py index 33d774e1..ac5078f3 100644 --- a/rsconnect/api.py +++ b/rsconnect/api.py @@ -385,7 +385,6 @@ def setup_remote_server( secret: str = None, ): validation.validate_connection_options( - name=name, url=url, api_key=api_key, insecure=insecure, @@ -393,6 +392,7 @@ def setup_remote_server( account_name=account_name, token=token, secret=secret, + name=name, ) if cacert and not ca_data: diff --git a/rsconnect/main.py b/rsconnect/main.py index 76fd7ffb..3ab5ff81 100644 --- a/rsconnect/main.py +++ b/rsconnect/main.py @@ -303,12 +303,12 @@ def add(name, server, api_key, insecure, cacert, account, token, secret, verbose set_verbosity(verbose) - validation.validate_new_server_options( + validation.validate_connection_options( url=server, api_key=api_key, insecure=insecure, cacert=cacert, - account=account, + account_name=account, token=token, secret=secret, ) diff --git a/rsconnect/validation.py b/rsconnect/validation.py index d8cdb999..a7ff2acb 100644 --- a/rsconnect/validation.py +++ b/rsconnect/validation.py @@ -7,11 +7,7 @@ def _get_present_options(options: typing.Dict[str, typing.Optional[str]]) -> typ return [k for k, v in options.items() if v] -def validate_new_server_options(url, api_key, insecure, cacert, account, token, secret): - validate_connection_options(None, url, api_key, insecure, cacert, account, token, secret) - - -def validate_connection_options(name, url, api_key, insecure, cacert, account_name, token, secret): +def validate_connection_options(url, api_key, insecure, cacert, account_name, token, secret, name=None): """ Validates provided Connect or shinyapps.io connection options and returns which target to use given the provided options.