diff --git a/docs/source/examples/file_server/app.py b/docs/source/examples/file_server/app.py index edb65f6f63d52..fa0a773779ad6 100644 --- a/docs/source/examples/file_server/app.py +++ b/docs/source/examples/file_server/app.py @@ -230,7 +230,7 @@ def test_file_server(): def test_file_server_in_cloud(): # You need to provide the directory containing the app file. app_dir = "docs/source/examples/file_server" - with run_app_in_cloud(app_dir) as (admin_page, view_page, get_logs_fn): + with run_app_in_cloud(app_dir) as (admin_page, view_page, get_logs_fn, name): """# 1. `admin_page` and `view_page` are playwright Page Objects. # Check out https://playwright.dev/python/ doc to learn more. diff --git a/src/lightning/app/CHANGELOG.md b/src/lightning/app/CHANGELOG.md index 16329df1a3382..f0c8fce8fcd6f 100644 --- a/src/lightning/app/CHANGELOG.md +++ b/src/lightning/app/CHANGELOG.md @@ -26,6 +26,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). - Removed support for Python 3.7 ([#16579](https://github.com/Lightning-AI/lightning/pull/16579)) +- Removed implicit ui testing with `testing.run_app_in_cloud` in favor of headless login and app selection ([#16741](https://github.com/Lightning-AI/lightning/pull/16741)) ### Fixed diff --git a/src/lightning/app/runners/cloud.py b/src/lightning/app/runners/cloud.py index ca5865e8d4296..9a21d63742b12 100644 --- a/src/lightning/app/runners/cloud.py +++ b/src/lightning/app/runners/cloud.py @@ -367,6 +367,10 @@ def dispatch( click.launch( self._get_app_url(project, run_instance, "logs" if run.is_headless else "web-ui", needs_credits) ) + + if bool(int(os.getenv("LIGHTING_TESTING", "0"))): + print(f"APP_LOGS_URL: {self._get_app_url(project, run_instance, 'logs')}") + except ApiException as e: logger.error(e.body) sys.exit(1) diff --git a/src/lightning/app/testing/testing.py b/src/lightning/app/testing/testing.py index dce3a0ac24e22..bcac812563559 100644 --- a/src/lightning/app/testing/testing.py +++ b/src/lightning/app/testing/testing.py @@ -37,7 +37,7 @@ from lightning.app import LightningApp, LightningFlow from lightning.app.cli.lightning_cli import run_app -from lightning.app.core.constants import LIGHTNING_CLOUD_PROJECT_ID +from lightning.app.core import constants from lightning.app.runners.multiprocess import MultiProcessRuntime from lightning.app.testing.config import _Config from lightning.app.utilities.app_logs import _app_logs_reader @@ -51,7 +51,6 @@ from lightning.app.utilities.proxies import ProxyWorkRun if _is_playwright_available(): - import playwright from playwright.sync_api import HttpCredentials, sync_playwright @@ -245,8 +244,8 @@ def run_app_in_cloud( ) -> Generator: """This utility is used to automate testing e2e application with lightning.ai.""" # 1. Validate the provide app_folder is correct. - if not os.path.exists(os.path.join(app_folder, "app.py")): - raise Exception("The app folder should contain an app.py file.") + if not os.path.exists(os.path.join(app_folder, app_name)): + raise Exception(f"The app folder should contain an {app_name} file.") if app_folder.endswith("/"): app_folder = app_folder[:-1] @@ -312,6 +311,14 @@ def run_app_in_cloud( process = Popen((cmd + extra_args), cwd=tmpdir, env=env_copy, stdout=stdout, stderr=sys.stderr) process.wait() + # Fallback URL to prevent failures in case we don't get the admin URL + admin_url = _Config.url + with open(stdout_path) as fo: + for line in fo.readlines(): + if line.startswith("APP_LOGS_URL: "): + admin_url = line.replace("APP_LOGS_URL: ", "") + break + if is_editable_mode: # Added to ensure the current code is properly uploaded. # Otherwise, it could result in un-tested PRs. @@ -338,11 +345,20 @@ def run_app_in_cloud( record_video_dir=os.path.join(_Config.video_location, TEST_APP_NAME), record_har_path=_Config.har_location, ) + + client = LightningClient() + project_id = _get_project(client).project_id + + app = _fetch_app_by_name(client, project_id, name) + app_id = app.id + print(f"The Lightning App ID is: {app.id}") # useful for Grafana + + if debug: + process = Process(target=_print_logs, kwargs={"app_id": app_id}) + process.start() + admin_page = context.new_page() - print(f"The Lightning App Token is: {token}") - print(f"The Lightning App user key is: {_Config.key}") - print(f"The Lightning App user id is: {_Config.id}") - admin_page.goto(_Config.url) + admin_page.goto(admin_url) admin_page.evaluate( """data => { window.localStorage.setItem('gridUserId', data[0]); @@ -352,59 +368,14 @@ def run_app_in_cloud( """, [_Config.id, _Config.key, token], ) - if LIGHTNING_CLOUD_PROJECT_ID: + if constants.LIGHTNING_CLOUD_PROJECT_ID: admin_page.evaluate( """data => { window.localStorage.setItem('gridDefaultProjectIdOverride', JSON.stringify(data[0])); } """, - [LIGHTNING_CLOUD_PROJECT_ID], + [constants.LIGHTNING_CLOUD_PROJECT_ID], ) - admin_page.goto(f"{_Config.url}/{_Config.username}/apps", timeout=60 * 1000) - - # Closing the Complete your profile dialog - try: - dialog = admin_page.locator("text=Complete your profile") - dialog.wait_for(timeout=10 * 1000, state="visible") - print("'Complete your profile' dialog visible, closing it.") - admin_page.locator('input[name="firstName"]').fill("first") - admin_page.locator('input[name="lastName"]').fill("last") - admin_page.locator('input[name="email"]').fill("e2e.test.admin@lightning.ai") - admin_page.locator('input[name="organization"]').fill("Lightning AI") - button = admin_page.locator('button:has-text("Confirm")') - button.wait_for(timeout=3 * 1000) - button.click() - except playwright._impl._api_types.TimeoutError: - print("'Complete your profile' dialog not visible, skipping.") - - # Closing the Create Project dialog. - try: - project_dialog = admin_page.locator("text=Create a project") - project_dialog.wait_for(timeout=10 * 1000, state="visible") - print("'Create Project' dialog visible, closing it.") - project_name_input = admin_page.locator('input[type="text"]') - project_name_input.fill("Default Project") - button = admin_page.locator('button:has-text("Continue")') - button.wait_for(timeout=3 * 1000) - button.click() - except playwright._impl._api_types.TimeoutError: - print("'Create Project' dialog not visible, skipping.") - - admin_page.locator(f'[data-cy="{name}"]').click() - - app_url = admin_page.url - admin_page.goto(app_url + "/logs") - - client = LightningClient() - project_id = _get_project(client).project_id - - app = _fetch_app_by_name(client, project_id, name) - app_id = app.id - print(f"The Lightning App ID is: {app.id}") # useful for Grafana - - if debug: - process = Process(target=_print_logs, kwargs={"app_id": app_id}) - process.start() view_page = context.new_page() i = 1 diff --git a/tests/integrations_app/public/test_commands_and_api.py b/tests/integrations_app/public/test_commands_and_api.py index 226b80bfd39f7..e7288cc125780 100644 --- a/tests/integrations_app/public/test_commands_and_api.py +++ b/tests/integrations_app/public/test_commands_and_api.py @@ -12,7 +12,7 @@ @pytest.mark.cloud def test_commands_and_api_example_cloud() -> None: with run_app_in_cloud(os.path.join(_PATH_EXAMPLES, "app_commands_and_api")) as ( - admin_page, + _, view_page, fetch_logs, app_name, diff --git a/tests/integrations_app/public/test_v0_app.py b/tests/integrations_app/public/test_v0_app.py index 885fa5b03e9a2..5cc011d15b427 100644 --- a/tests/integrations_app/public/test_v0_app.py +++ b/tests/integrations_app/public/test_v0_app.py @@ -67,11 +67,7 @@ def test_v0_app_example_byoc_cloud() -> None: with run_app_in_cloud( os.path.join(_PATH_EXAMPLES, "app_v0"), extra_args=["--cluster-id", os.environ.get("LIGHTNING_BYOC_CLUSTER_ID")], - ) as ( - _, - view_page, - fetch_logs, - ): + ) as (_, view_page, fetch_logs, app_name): run_v0_app(fetch_logs, view_page)