From 69e69151ac167951c0d06ae87abe0ae0893e3b3e Mon Sep 17 00:00:00 2001
From: Justus Schock <justus.schock@posteo.de>
Date: Mon, 13 Feb 2023 13:34:58 +0100
Subject: [PATCH 01/14] use app_name for existence check

---
 src/lightning/app/testing/testing.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/lightning/app/testing/testing.py b/src/lightning/app/testing/testing.py
index f37e4b9853600..0e9e76805bdf2 100644
--- a/src/lightning/app/testing/testing.py
+++ b/src/lightning/app/testing/testing.py
@@ -245,8 +245,8 @@ def run_app_in_cloud(
 ) -> Generator:
     """This utility is used to automate testing e2e application with lightning.ai."""
     # 1. Validate the provide app_folder is correct.
-    if not os.path.exists(os.path.join(app_folder, "app.py")):
-        raise Exception("The app folder should contain an app.py file.")
+    if not os.path.exists(os.path.join(app_folder, app_name)):
+        raise Exception(f"The app folder should contain an {app_name} file.")
     if app_folder.endswith("/"):
         app_folder = app_folder[:-1]
 

From 9909394ec5d31db96cc10b4882064fd411904b2b Mon Sep 17 00:00:00 2001
From: Justus Schock <justus.schock@posteo.de>
Date: Mon, 13 Feb 2023 13:35:58 +0100
Subject: [PATCH 02/14] remove token request

---
 src/lightning/app/testing/testing.py | 13 -------------
 1 file changed, 13 deletions(-)

diff --git a/src/lightning/app/testing/testing.py b/src/lightning/app/testing/testing.py
index 0e9e76805bdf2..c9efe7546e85f 100644
--- a/src/lightning/app/testing/testing.py
+++ b/src/lightning/app/testing/testing.py
@@ -268,19 +268,6 @@ def run_app_in_cloud(
 
     os.environ["LIGHTNING_APP_NAME"] = name
 
-    url = _Config.url
-    if url.endswith("/"):
-        url = url[:-1]
-    payload = {"apiKey": _Config.api_key, "username": _Config.username}
-    url_login = url + "/v1/auth/login"
-    res = requests.post(url_login, data=json.dumps(payload))
-    if "token" not in res.json():
-        raise RuntimeError(
-            f"You haven't properly setup your environment variables with {url_login} and data: \n{payload}"
-        )
-
-    token = res.json()["token"]
-
     # 3. Disconnect from the App if any.
     Popen("lightning disconnect", shell=True).wait()
 

From dfcfba1c1f2ec781817892e813cd1e6daa4d747f Mon Sep 17 00:00:00 2001
From: Justus Schock <justus.schock@posteo.de>
Date: Mon, 13 Feb 2023 13:37:16 +0100
Subject: [PATCH 03/14] remove implicit ui testing

---
 src/lightning/app/testing/testing.py | 53 ----------------------------
 1 file changed, 53 deletions(-)

diff --git a/src/lightning/app/testing/testing.py b/src/lightning/app/testing/testing.py
index c9efe7546e85f..e2ddf9e829d30 100644
--- a/src/lightning/app/testing/testing.py
+++ b/src/lightning/app/testing/testing.py
@@ -325,59 +325,6 @@ def run_app_in_cloud(
             record_video_dir=os.path.join(_Config.video_location, TEST_APP_NAME),
             record_har_path=_Config.har_location,
         )
-        admin_page = context.new_page()
-        print(f"The Lightning App Token is: {token}")
-        print(f"The Lightning App user key is: {_Config.key}")
-        print(f"The Lightning App user id is: {_Config.id}")
-        admin_page.goto(_Config.url)
-        admin_page.evaluate(
-            """data => {
-            window.localStorage.setItem('gridUserId', data[0]);
-            window.localStorage.setItem('gridUserKey', data[1]);
-            window.localStorage.setItem('gridUserToken', data[2]);
-        }
-        """,
-            [_Config.id, _Config.key, token],
-        )
-        if LIGHTNING_CLOUD_PROJECT_ID:
-            admin_page.evaluate(
-                """data => {
-                window.localStorage.setItem('gridDefaultProjectIdOverride', JSON.stringify(data[0]));
-            }
-            """,
-                [LIGHTNING_CLOUD_PROJECT_ID],
-            )
-        admin_page.goto(f"{_Config.url}/{_Config.username}/apps", timeout=60 * 1000)
-
-        # Closing the Complete your profile dialog
-        try:
-            dialog = admin_page.locator("text=Complete your profile")
-            dialog.wait_for(timeout=10 * 1000, state="visible")
-            print("'Complete your profile' dialog visible, closing it.")
-            admin_page.locator('input[name="firstName"]').fill("first")
-            admin_page.locator('input[name="lastName"]').fill("last")
-            admin_page.locator('input[name="email"]').fill("e2e.test.admin@lightning.ai")
-            admin_page.locator('input[name="organization"]').fill("Lightning AI")
-            button = admin_page.locator('button:has-text("Confirm")')
-            button.wait_for(timeout=3 * 1000)
-            button.click()
-        except playwright._impl._api_types.TimeoutError:
-            print("'Complete your profile' dialog not visible, skipping.")
-
-        # Closing the Create Project dialog.
-        try:
-            project_dialog = admin_page.locator("text=Create a project")
-            project_dialog.wait_for(timeout=10 * 1000, state="visible")
-            print("'Create Project' dialog visible, closing it.")
-            project_name_input = admin_page.locator('input[type="text"]')
-            project_name_input.fill("Default Project")
-            button = admin_page.locator('button:has-text("Continue")')
-            button.wait_for(timeout=3 * 1000)
-            button.click()
-        except playwright._impl._api_types.TimeoutError:
-            print("'Create Project' dialog not visible, skipping.")
-
-        admin_page.locator(f'[data-cy="{name}"]').click()
 
         client = LightningClient()
         project_id = _get_project(client).project_id

From 0b53053ac1311f4b1e73bbe2b3b38772585300ea Mon Sep 17 00:00:00 2001
From: Justus Schock <justus.schock@posteo.de>
Date: Mon, 13 Feb 2023 13:39:23 +0100
Subject: [PATCH 04/14] remove credentials for headless browser login

---
 src/lightning/app/testing/testing.py | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/src/lightning/app/testing/testing.py b/src/lightning/app/testing/testing.py
index e2ddf9e829d30..f301e542955d0 100644
--- a/src/lightning/app/testing/testing.py
+++ b/src/lightning/app/testing/testing.py
@@ -318,10 +318,6 @@ def run_app_in_cloud(
     with sync_playwright() as p:
         browser = p.chromium.launch(headless=bool(int(os.getenv("HEADLESS", "0"))))
         context = browser.new_context(
-            # Eventually this will need to be deleted
-            http_credentials=HttpCredentials(
-                {"username": os.getenv("LAI_USER", "").strip(), "password": os.getenv("LAI_PASS", "")}
-            ),
             record_video_dir=os.path.join(_Config.video_location, TEST_APP_NAME),
             record_har_path=_Config.har_location,
         )

From 1ed25034c831b8cc0c0b8fd4b4d7922d41c9e804 Mon Sep 17 00:00:00 2001
From: Justus Schock <justus.schock@posteo.de>
Date: Mon, 13 Feb 2023 13:40:20 +0100
Subject: [PATCH 05/14] don't yield admin_page

---
 src/lightning/app/testing/testing.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/lightning/app/testing/testing.py b/src/lightning/app/testing/testing.py
index f301e542955d0..a66a3e349ebb7 100644
--- a/src/lightning/app/testing/testing.py
+++ b/src/lightning/app/testing/testing.py
@@ -391,7 +391,7 @@ def add_prefix(c: str) -> str:
                 yield log_event.message
 
         try:
-            yield admin_page, view_page, fetch_logs, name
+            yield view_page, fetch_logs, name
         except KeyboardInterrupt:
             pass
         finally:

From bf34d0fdc13c38f20c3b323d1b7072a5999cbfd0 Mon Sep 17 00:00:00 2001
From: Justus Schock <justus.schock@posteo.de>
Date: Mon, 13 Feb 2023 13:46:44 +0100
Subject: [PATCH 06/14] update usage

---
 docs/source/examples/file_server/app.py                       | 4 ++--
 tests/integrations_app/flagship/test_flashy.py                | 2 +-
 tests/integrations_app/local/test_collect_failures.py         | 1 -
 tests/integrations_app/local/test_custom_work_dependencies.py | 2 +-
 tests/integrations_app/local/test_idle_timeout.py             | 1 -
 tests/integrations_app/public/test_app_dag.py                 | 2 +-
 tests/integrations_app/public/test_boring_app.py              | 1 -
 tests/integrations_app/public/test_commands_and_api.py        | 1 -
 tests/integrations_app/public/test_drive.py                   | 1 -
 .../integrations_app/public/test_installation_commands_app.py | 2 +-
 tests/integrations_app/public/test_payload.py                 | 2 +-
 tests/integrations_app/public/test_quick_start.py             | 2 +-
 tests/integrations_app/public/test_template_react_ui.py       | 1 -
 tests/integrations_app/public/test_template_streamlit_ui.py   | 1 -
 tests/integrations_app/public/test_v0_app.py                  | 3 +--
 15 files changed, 9 insertions(+), 17 deletions(-)

diff --git a/docs/source/examples/file_server/app.py b/docs/source/examples/file_server/app.py
index edb65f6f63d52..90b98e1668ec4 100644
--- a/docs/source/examples/file_server/app.py
+++ b/docs/source/examples/file_server/app.py
@@ -230,8 +230,8 @@ def test_file_server():
 def test_file_server_in_cloud():
     # You need to provide the directory containing the app file.
     app_dir = "docs/source/examples/file_server"
-    with run_app_in_cloud(app_dir) as (admin_page, view_page, get_logs_fn):
-        """# 1. `admin_page` and `view_page` are playwright Page Objects.
+    with run_app_in_cloud(app_dir) as (view_page, get_logs_fn, name):
+        """# 1. `view_page` is a playwright Page Objects.
 
         # Check out https://playwright.dev/python/ doc to learn more.
         # You can click the UI and trigger actions.
diff --git a/tests/integrations_app/flagship/test_flashy.py b/tests/integrations_app/flagship/test_flashy.py
index 8a593deeb5282..046b693819ca9 100644
--- a/tests/integrations_app/flagship/test_flashy.py
+++ b/tests/integrations_app/flagship/test_flashy.py
@@ -73,6 +73,6 @@ def validate_app_functionalities(app_page: "Page") -> None:
 
 @pytest.mark.cloud
 def test_app_cloud() -> None:
-    with run_app_in_cloud(_PATH_INTEGRATIONS_DIR) as (admin_page, view_page, fetch_logs, _):
+    with run_app_in_cloud(_PATH_INTEGRATIONS_DIR) as (view_page, fetch_logs, _):
 
         validate_app_functionalities(view_page)
diff --git a/tests/integrations_app/local/test_collect_failures.py b/tests/integrations_app/local/test_collect_failures.py
index 22e3c9680cf8e..47d488758e96b 100644
--- a/tests/integrations_app/local/test_collect_failures.py
+++ b/tests/integrations_app/local/test_collect_failures.py
@@ -23,7 +23,6 @@ def test_collect_failures_example_cloud() -> None:
         "invalid_value_of_i_5",
     ]
     with run_app_in_cloud(os.path.join(_PATH_APPS, "collect_failures")) as (
-        _,
         _,
         fetch_logs,
         _,
diff --git a/tests/integrations_app/local/test_custom_work_dependencies.py b/tests/integrations_app/local/test_custom_work_dependencies.py
index 33400d7c96e81..f0fe338a8aefd 100644
--- a/tests/integrations_app/local/test_custom_work_dependencies.py
+++ b/tests/integrations_app/local/test_custom_work_dependencies.py
@@ -13,7 +13,7 @@ def test_custom_work_dependencies_example_cloud() -> None:
     with run_app_in_cloud(
         os.path.join(_PATH_APPS, "custom_work_dependencies"),
         app_name="app.py",
-    ) as (_, _, fetch_logs, _):
+    ) as (_, fetch_logs, _):
         has_logs = False
         while not has_logs:
             for log in fetch_logs(["flow"]):
diff --git a/tests/integrations_app/local/test_idle_timeout.py b/tests/integrations_app/local/test_idle_timeout.py
index ba0daf2dc379c..35a6d42861725 100644
--- a/tests/integrations_app/local/test_idle_timeout.py
+++ b/tests/integrations_app/local/test_idle_timeout.py
@@ -10,7 +10,6 @@
 @pytest.mark.cloud
 def test_idle_timeout_example_cloud() -> None:
     with run_app_in_cloud(os.path.join(_PATH_APPS, "idle_timeout")) as (
-        _,
         _,
         fetch_logs,
         _,
diff --git a/tests/integrations_app/public/test_app_dag.py b/tests/integrations_app/public/test_app_dag.py
index 4c1ba3348b2f1..eac1d99939fad 100644
--- a/tests/integrations_app/public/test_app_dag.py
+++ b/tests/integrations_app/public/test_app_dag.py
@@ -9,7 +9,7 @@
 
 @pytest.mark.cloud
 def test_app_dag_example_cloud() -> None:
-    with run_app_in_cloud(os.path.join(_PATH_EXAMPLES, "app_dag")) as (_, _, fetch_logs, _):
+    with run_app_in_cloud(os.path.join(_PATH_EXAMPLES, "app_dag")) as (_, fetch_logs, _):
 
         launch_log, finish_log = False, False
         while not (launch_log and finish_log):
diff --git a/tests/integrations_app/public/test_boring_app.py b/tests/integrations_app/public/test_boring_app.py
index 49273918098bb..5ec2b170a7df5 100644
--- a/tests/integrations_app/public/test_boring_app.py
+++ b/tests/integrations_app/public/test_boring_app.py
@@ -11,7 +11,6 @@
 @pytest.mark.cloud
 def test_boring_app_example_cloud() -> None:
     with run_app_in_cloud(os.path.join(_PATH_EXAMPLES, "app_boring"), app_name="app_dynamic.py", debug=True) as (
-        _,
         view_page,
         _,
         name,
diff --git a/tests/integrations_app/public/test_commands_and_api.py b/tests/integrations_app/public/test_commands_and_api.py
index 16e9b071b8d8f..991782a32fce2 100644
--- a/tests/integrations_app/public/test_commands_and_api.py
+++ b/tests/integrations_app/public/test_commands_and_api.py
@@ -12,7 +12,6 @@
 @pytest.mark.cloud
 def test_commands_and_api_example_cloud() -> None:
     with run_app_in_cloud(os.path.join(_PATH_EXAMPLES, "app_commands_and_api")) as (
-        admin_page,
         view_page,
         fetch_logs,
         app_name,
diff --git a/tests/integrations_app/public/test_drive.py b/tests/integrations_app/public/test_drive.py
index 757cc66ec5d95..579b1587a341f 100644
--- a/tests/integrations_app/public/test_drive.py
+++ b/tests/integrations_app/public/test_drive.py
@@ -10,7 +10,6 @@
 @pytest.mark.cloud
 def test_drive_example_cloud() -> None:
     with run_app_in_cloud(os.path.join(_PATH_EXAMPLES, "app_drive")) as (
-        _,
         _,
         fetch_logs,
         _,
diff --git a/tests/integrations_app/public/test_installation_commands_app.py b/tests/integrations_app/public/test_installation_commands_app.py
index 7dbdf932ac650..71d6ac7343669 100644
--- a/tests/integrations_app/public/test_installation_commands_app.py
+++ b/tests/integrations_app/public/test_installation_commands_app.py
@@ -14,7 +14,7 @@ def test_installation_commands_app_example_cloud() -> None:
         app_name="app.py",
         extra_args=["--setup"],
         debug=True,
-    ) as (_, _, fetch_logs, _):
+    ) as (_, fetch_logs, _):
         has_logs = False
         while not has_logs:
             for log in fetch_logs(["work"]):
diff --git a/tests/integrations_app/public/test_payload.py b/tests/integrations_app/public/test_payload.py
index e4b85c0f180c5..32daf71d609e3 100644
--- a/tests/integrations_app/public/test_payload.py
+++ b/tests/integrations_app/public/test_payload.py
@@ -9,7 +9,7 @@
 
 @pytest.mark.cloud
 def test_payload_example_cloud() -> None:
-    with run_app_in_cloud(os.path.join(_PATH_EXAMPLES, "app_payload")) as (_, _, fetch_logs, _):
+    with run_app_in_cloud(os.path.join(_PATH_EXAMPLES, "app_payload")) as (_, fetch_logs, _):
 
         has_logs = False
         while not has_logs:
diff --git a/tests/integrations_app/public/test_quick_start.py b/tests/integrations_app/public/test_quick_start.py
index 29cae720278bb..4c59a516248f0 100644
--- a/tests/integrations_app/public/test_quick_start.py
+++ b/tests/integrations_app/public/test_quick_start.py
@@ -51,7 +51,7 @@ def test_quick_start_example(caplog, monkeypatch):
 
 @pytest.mark.cloud
 def test_quick_start_example_cloud() -> None:
-    with run_app_in_cloud(os.path.join(_PATH_EXAMPLES, "lightning-quick-start")) as (_, view_page, _, _):
+    with run_app_in_cloud(os.path.join(_PATH_EXAMPLES, "lightning-quick-start")) as (view_page, _, _):
 
         def click_gradio_demo(*_, **__):
             button = view_page.locator('button:has-text("Interactive demo")')
diff --git a/tests/integrations_app/public/test_template_react_ui.py b/tests/integrations_app/public/test_template_react_ui.py
index 5cb07510f24bc..fb968dace44bc 100644
--- a/tests/integrations_app/public/test_template_react_ui.py
+++ b/tests/integrations_app/public/test_template_react_ui.py
@@ -11,7 +11,6 @@
 def test_template_react_ui_example_cloud() -> None:
     """This test ensures streamlit works in the cloud by clicking a button and checking the logs."""
     with run_app_in_cloud(os.path.join(_PATH_EXAMPLES, "app_template_react_ui")) as (
-        _,
         view_page,
         fetch_logs,
         _,
diff --git a/tests/integrations_app/public/test_template_streamlit_ui.py b/tests/integrations_app/public/test_template_streamlit_ui.py
index 6ca7cd9fd7535..f3a03622d1498 100644
--- a/tests/integrations_app/public/test_template_streamlit_ui.py
+++ b/tests/integrations_app/public/test_template_streamlit_ui.py
@@ -11,7 +11,6 @@
 def test_template_streamlit_ui_example_cloud() -> None:
     """This test ensures streamlit works in the cloud by clicking a button and checking the logs."""
     with run_app_in_cloud(os.path.join(_PATH_EXAMPLES, "app_template_streamlit_ui")) as (
-        _,
         view_page,
         fetch_logs,
         _,
diff --git a/tests/integrations_app/public/test_v0_app.py b/tests/integrations_app/public/test_v0_app.py
index 885fa5b03e9a2..ad43aaec4ee9e 100644
--- a/tests/integrations_app/public/test_v0_app.py
+++ b/tests/integrations_app/public/test_v0_app.py
@@ -68,9 +68,9 @@ def test_v0_app_example_byoc_cloud() -> None:
         os.path.join(_PATH_EXAMPLES, "app_v0"),
         extra_args=["--cluster-id", os.environ.get("LIGHTNING_BYOC_CLUSTER_ID")],
     ) as (
-        _,
         view_page,
         fetch_logs,
+        app_name
     ):
         run_v0_app(fetch_logs, view_page)
 
@@ -78,7 +78,6 @@ def test_v0_app_example_byoc_cloud() -> None:
 @pytest.mark.cloud
 def test_v0_app_example_cloud() -> None:
     with run_app_in_cloud(os.path.join(_PATH_EXAMPLES, "app_v0")) as (
-        _,
         view_page,
         fetch_logs,
         _,

From d769232e0c84f40d2774d9f259a0a9a28424c910 Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Mon, 13 Feb 2023 12:50:02 +0000
Subject: [PATCH 07/14] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 src/lightning/app/testing/testing.py         | 5 +----
 tests/integrations_app/public/test_v0_app.py | 6 +-----
 2 files changed, 2 insertions(+), 9 deletions(-)

diff --git a/src/lightning/app/testing/testing.py b/src/lightning/app/testing/testing.py
index a66a3e349ebb7..595ef3a1fd907 100644
--- a/src/lightning/app/testing/testing.py
+++ b/src/lightning/app/testing/testing.py
@@ -14,7 +14,6 @@
 
 import asyncio
 import datetime
-import json
 import os
 import shutil
 import subprocess
@@ -37,7 +36,6 @@
 
 from lightning.app import LightningApp, LightningFlow
 from lightning.app.cli.lightning_cli import run_app
-from lightning.app.core.constants import LIGHTNING_CLOUD_PROJECT_ID
 from lightning.app.runners.multiprocess import MultiProcessRuntime
 from lightning.app.testing.config import _Config
 from lightning.app.utilities.app_logs import _app_logs_reader
@@ -51,8 +49,7 @@
 from lightning.app.utilities.proxies import ProxyWorkRun
 
 if _is_playwright_available():
-    import playwright
-    from playwright.sync_api import HttpCredentials, sync_playwright
+    from playwright.sync_api import sync_playwright
 
 
 def _on_error_callback(ws_app, *_):
diff --git a/tests/integrations_app/public/test_v0_app.py b/tests/integrations_app/public/test_v0_app.py
index ad43aaec4ee9e..f36c584a5df93 100644
--- a/tests/integrations_app/public/test_v0_app.py
+++ b/tests/integrations_app/public/test_v0_app.py
@@ -67,11 +67,7 @@ def test_v0_app_example_byoc_cloud() -> None:
     with run_app_in_cloud(
         os.path.join(_PATH_EXAMPLES, "app_v0"),
         extra_args=["--cluster-id", os.environ.get("LIGHTNING_BYOC_CLUSTER_ID")],
-    ) as (
-        view_page,
-        fetch_logs,
-        app_name
-    ):
+    ) as (view_page, fetch_logs, app_name):
         run_v0_app(fetch_logs, view_page)
 
 

From 49807e9e66004a343793a00d09f730bf59c6e92f Mon Sep 17 00:00:00 2001
From: Justus Schock <justus.schock@posteo.de>
Date: Mon, 13 Feb 2023 13:50:59 +0100
Subject: [PATCH 08/14] update changelog

---
 src/lightning/app/CHANGELOG.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/lightning/app/CHANGELOG.md b/src/lightning/app/CHANGELOG.md
index 6320b66d13602..c431cbe671713 100644
--- a/src/lightning/app/CHANGELOG.md
+++ b/src/lightning/app/CHANGELOG.md
@@ -26,6 +26,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
 
 - Removed support for Python 3.7 ([#16579](https://github.com/Lightning-AI/lightning/pull/16579))
 
+- Removed implicit ui testing with `testing.run_app_in_cloud` in favor of headless login and app selection ([#16741](https://github.com/Lightning-AI/lightning/pull/16741))
 
 ### Fixed
 

From 8316563d1fcf9e71d4e4c40b680ad986afaf0fdb Mon Sep 17 00:00:00 2001
From: Justus Schock <justus.schock@posteo.de>
Date: Mon, 13 Feb 2023 13:39:23 +0100
Subject: [PATCH 09/14] Revert "remove credentials for headless browser login"

This reverts commit 0b53053ac1311f4b1e73bbe2b3b38772585300ea.
---
 src/lightning/app/testing/testing.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/lightning/app/testing/testing.py b/src/lightning/app/testing/testing.py
index 595ef3a1fd907..369b7416eee1e 100644
--- a/src/lightning/app/testing/testing.py
+++ b/src/lightning/app/testing/testing.py
@@ -315,6 +315,10 @@ def run_app_in_cloud(
     with sync_playwright() as p:
         browser = p.chromium.launch(headless=bool(int(os.getenv("HEADLESS", "0"))))
         context = browser.new_context(
+            # Eventually this will need to be deleted
+            http_credentials=HttpCredentials(
+                {"username": os.getenv("LAI_USER", "").strip(), "password": os.getenv("LAI_PASS", "")}
+            ),
             record_video_dir=os.path.join(_Config.video_location, TEST_APP_NAME),
             record_har_path=_Config.har_location,
         )

From 136042c42d643e72e25dadd4eaf12087572f27f2 Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Mon, 13 Feb 2023 12:50:02 +0000
Subject: [PATCH 10/14] Revert "[pre-commit.ci] auto fixes from pre-commit.com
 hooks"

This reverts commit d769232e0c84f40d2774d9f259a0a9a28424c910.
---
 src/lightning/app/testing/testing.py         | 5 ++++-
 tests/integrations_app/public/test_v0_app.py | 6 +++++-
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/src/lightning/app/testing/testing.py b/src/lightning/app/testing/testing.py
index 369b7416eee1e..f06619c1c4c9a 100644
--- a/src/lightning/app/testing/testing.py
+++ b/src/lightning/app/testing/testing.py
@@ -14,6 +14,7 @@
 
 import asyncio
 import datetime
+import json
 import os
 import shutil
 import subprocess
@@ -36,6 +37,7 @@
 
 from lightning.app import LightningApp, LightningFlow
 from lightning.app.cli.lightning_cli import run_app
+from lightning.app.core.constants import LIGHTNING_CLOUD_PROJECT_ID
 from lightning.app.runners.multiprocess import MultiProcessRuntime
 from lightning.app.testing.config import _Config
 from lightning.app.utilities.app_logs import _app_logs_reader
@@ -49,7 +51,8 @@
 from lightning.app.utilities.proxies import ProxyWorkRun
 
 if _is_playwright_available():
-    from playwright.sync_api import sync_playwright
+    import playwright
+    from playwright.sync_api import HttpCredentials, sync_playwright
 
 
 def _on_error_callback(ws_app, *_):
diff --git a/tests/integrations_app/public/test_v0_app.py b/tests/integrations_app/public/test_v0_app.py
index f36c584a5df93..ad43aaec4ee9e 100644
--- a/tests/integrations_app/public/test_v0_app.py
+++ b/tests/integrations_app/public/test_v0_app.py
@@ -67,7 +67,11 @@ def test_v0_app_example_byoc_cloud() -> None:
     with run_app_in_cloud(
         os.path.join(_PATH_EXAMPLES, "app_v0"),
         extra_args=["--cluster-id", os.environ.get("LIGHTNING_BYOC_CLUSTER_ID")],
-    ) as (view_page, fetch_logs, app_name):
+    ) as (
+        view_page,
+        fetch_logs,
+        app_name
+    ):
         run_v0_app(fetch_logs, view_page)
 
 

From e8c8ea1a4f162aabeab5fea6b824761f2d207efd Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Mon, 13 Feb 2023 13:57:13 +0000
Subject: [PATCH 11/14] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 src/lightning/app/testing/testing.py         | 3 ---
 tests/integrations_app/public/test_v0_app.py | 6 +-----
 2 files changed, 1 insertion(+), 8 deletions(-)

diff --git a/src/lightning/app/testing/testing.py b/src/lightning/app/testing/testing.py
index f06619c1c4c9a..e1ca5aefa5c92 100644
--- a/src/lightning/app/testing/testing.py
+++ b/src/lightning/app/testing/testing.py
@@ -14,7 +14,6 @@
 
 import asyncio
 import datetime
-import json
 import os
 import shutil
 import subprocess
@@ -37,7 +36,6 @@
 
 from lightning.app import LightningApp, LightningFlow
 from lightning.app.cli.lightning_cli import run_app
-from lightning.app.core.constants import LIGHTNING_CLOUD_PROJECT_ID
 from lightning.app.runners.multiprocess import MultiProcessRuntime
 from lightning.app.testing.config import _Config
 from lightning.app.utilities.app_logs import _app_logs_reader
@@ -51,7 +49,6 @@
 from lightning.app.utilities.proxies import ProxyWorkRun
 
 if _is_playwright_available():
-    import playwright
     from playwright.sync_api import HttpCredentials, sync_playwright
 
 
diff --git a/tests/integrations_app/public/test_v0_app.py b/tests/integrations_app/public/test_v0_app.py
index ad43aaec4ee9e..f36c584a5df93 100644
--- a/tests/integrations_app/public/test_v0_app.py
+++ b/tests/integrations_app/public/test_v0_app.py
@@ -67,11 +67,7 @@ def test_v0_app_example_byoc_cloud() -> None:
     with run_app_in_cloud(
         os.path.join(_PATH_EXAMPLES, "app_v0"),
         extra_args=["--cluster-id", os.environ.get("LIGHTNING_BYOC_CLUSTER_ID")],
-    ) as (
-        view_page,
-        fetch_logs,
-        app_name
-    ):
+    ) as (view_page, fetch_logs, app_name):
         run_v0_app(fetch_logs, view_page)
 
 

From d426ddd82e9c8ae7f63d084bddde1a8d263692ae Mon Sep 17 00:00:00 2001
From: Ethan Harris <ethanwharris@gmail.com>
Date: Tue, 21 Feb 2023 22:21:11 +0000
Subject: [PATCH 12/14] Try fix

---
 src/lightning/app/testing/testing.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/src/lightning/app/testing/testing.py b/src/lightning/app/testing/testing.py
index f2e973471766c..d7a44b01be760 100644
--- a/src/lightning/app/testing/testing.py
+++ b/src/lightning/app/testing/testing.py
@@ -323,9 +323,6 @@ def run_app_in_cloud(
             record_har_path=_Config.har_location,
         )
 
-        app_url = admin_page.url
-        admin_page.goto(app_url + "/logs")
-
         client = LightningClient()
         project_id = _get_project(client).project_id
 

From 9b3d34942935ebf1f5dbea8df0fa6ce69965a2f8 Mon Sep 17 00:00:00 2001
From: Ethan Harris <ethanwharris@gmail.com>
Date: Wed, 22 Feb 2023 10:38:48 +0000
Subject: [PATCH 13/14] Add back an admin page without the implicit testing

---
 docs/source/examples/file_server/app.py       |  4 +-
 src/lightning/app/runners/cloud.py            |  4 ++
 src/lightning/app/testing/testing.py          | 45 ++++++++++++++++++-
 .../integrations_app/flagship/test_flashy.py  |  2 +-
 .../local/test_collect_failures.py            |  1 +
 .../local/test_custom_work_dependencies.py    |  2 +-
 .../local/test_idle_timeout.py                |  1 +
 tests/integrations_app/public/test_app_dag.py |  2 +-
 .../public/test_boring_app.py                 |  1 +
 .../public/test_commands_and_api.py           |  1 +
 tests/integrations_app/public/test_drive.py   |  1 +
 .../public/test_installation_commands_app.py  |  2 +-
 tests/integrations_app/public/test_payload.py |  2 +-
 .../public/test_quick_start.py                |  2 +-
 .../public/test_template_react_ui.py          |  1 +
 .../public/test_template_streamlit_ui.py      |  1 +
 tests/integrations_app/public/test_v0_app.py  |  3 +-
 17 files changed, 65 insertions(+), 10 deletions(-)

diff --git a/docs/source/examples/file_server/app.py b/docs/source/examples/file_server/app.py
index 90b98e1668ec4..fa0a773779ad6 100644
--- a/docs/source/examples/file_server/app.py
+++ b/docs/source/examples/file_server/app.py
@@ -230,8 +230,8 @@ def test_file_server():
 def test_file_server_in_cloud():
     # You need to provide the directory containing the app file.
     app_dir = "docs/source/examples/file_server"
-    with run_app_in_cloud(app_dir) as (view_page, get_logs_fn, name):
-        """# 1. `view_page` is a playwright Page Objects.
+    with run_app_in_cloud(app_dir) as (admin_page, view_page, get_logs_fn, name):
+        """# 1. `admin_page` and `view_page` are playwright Page Objects.
 
         # Check out https://playwright.dev/python/ doc to learn more.
         # You can click the UI and trigger actions.
diff --git a/src/lightning/app/runners/cloud.py b/src/lightning/app/runners/cloud.py
index ca5865e8d4296..9a21d63742b12 100644
--- a/src/lightning/app/runners/cloud.py
+++ b/src/lightning/app/runners/cloud.py
@@ -367,6 +367,10 @@ def dispatch(
                 click.launch(
                     self._get_app_url(project, run_instance, "logs" if run.is_headless else "web-ui", needs_credits)
                 )
+
+            if bool(int(os.getenv("LIGHTING_TESTING", "0"))):
+                print(f"APP_LOGS_URL: {self._get_app_url(project, run_instance, 'logs')}")
+
         except ApiException as e:
             logger.error(e.body)
             sys.exit(1)
diff --git a/src/lightning/app/testing/testing.py b/src/lightning/app/testing/testing.py
index d7a44b01be760..bcac812563559 100644
--- a/src/lightning/app/testing/testing.py
+++ b/src/lightning/app/testing/testing.py
@@ -14,6 +14,7 @@
 
 import asyncio
 import datetime
+import json
 import os
 import shutil
 import subprocess
@@ -36,6 +37,7 @@
 
 from lightning.app import LightningApp, LightningFlow
 from lightning.app.cli.lightning_cli import run_app
+from lightning.app.core import constants
 from lightning.app.runners.multiprocess import MultiProcessRuntime
 from lightning.app.testing.config import _Config
 from lightning.app.utilities.app_logs import _app_logs_reader
@@ -265,6 +267,19 @@ def run_app_in_cloud(
 
     os.environ["LIGHTNING_APP_NAME"] = name
 
+    url = _Config.url
+    if url.endswith("/"):
+        url = url[:-1]
+    payload = {"apiKey": _Config.api_key, "username": _Config.username}
+    url_login = url + "/v1/auth/login"
+    res = requests.post(url_login, data=json.dumps(payload))
+    if "token" not in res.json():
+        raise RuntimeError(
+            f"You haven't properly setup your environment variables with {url_login} and data: \n{payload}"
+        )
+
+    token = res.json()["token"]
+
     # 3. Disconnect from the App if any.
     Popen("lightning logout", shell=True).wait()
 
@@ -296,6 +311,14 @@ def run_app_in_cloud(
             process = Popen((cmd + extra_args), cwd=tmpdir, env=env_copy, stdout=stdout, stderr=sys.stderr)
             process.wait()
 
+        # Fallback URL to prevent failures in case we don't get the admin URL
+        admin_url = _Config.url
+        with open(stdout_path) as fo:
+            for line in fo.readlines():
+                if line.startswith("APP_LOGS_URL: "):
+                    admin_url = line.replace("APP_LOGS_URL: ", "")
+                    break
+
         if is_editable_mode:
             # Added to ensure the current code is properly uploaded.
             # Otherwise, it could result in un-tested PRs.
@@ -334,6 +357,26 @@ def run_app_in_cloud(
             process = Process(target=_print_logs, kwargs={"app_id": app_id})
             process.start()
 
+        admin_page = context.new_page()
+        admin_page.goto(admin_url)
+        admin_page.evaluate(
+            """data => {
+            window.localStorage.setItem('gridUserId', data[0]);
+            window.localStorage.setItem('gridUserKey', data[1]);
+            window.localStorage.setItem('gridUserToken', data[2]);
+        }
+        """,
+            [_Config.id, _Config.key, token],
+        )
+        if constants.LIGHTNING_CLOUD_PROJECT_ID:
+            admin_page.evaluate(
+                """data => {
+                window.localStorage.setItem('gridDefaultProjectIdOverride', JSON.stringify(data[0]));
+            }
+            """,
+                [constants.LIGHTNING_CLOUD_PROJECT_ID],
+            )
+
         view_page = context.new_page()
         i = 1
         while True:
@@ -392,7 +435,7 @@ def add_prefix(c: str) -> str:
                 yield log_event.message
 
         try:
-            yield view_page, fetch_logs, name
+            yield admin_page, view_page, fetch_logs, name
         except KeyboardInterrupt:
             pass
         finally:
diff --git a/tests/integrations_app/flagship/test_flashy.py b/tests/integrations_app/flagship/test_flashy.py
index 046b693819ca9..f680d5d86e371 100644
--- a/tests/integrations_app/flagship/test_flashy.py
+++ b/tests/integrations_app/flagship/test_flashy.py
@@ -73,6 +73,6 @@ def validate_app_functionalities(app_page: "Page") -> None:
 
 @pytest.mark.cloud
 def test_app_cloud() -> None:
-    with run_app_in_cloud(_PATH_INTEGRATIONS_DIR) as (view_page, fetch_logs, _):
+    with run_app_in_cloud(_PATH_INTEGRATIONS_DIR) as (_, view_page, fetch_logs, _):
 
         validate_app_functionalities(view_page)
diff --git a/tests/integrations_app/local/test_collect_failures.py b/tests/integrations_app/local/test_collect_failures.py
index 47d488758e96b..22e3c9680cf8e 100644
--- a/tests/integrations_app/local/test_collect_failures.py
+++ b/tests/integrations_app/local/test_collect_failures.py
@@ -23,6 +23,7 @@ def test_collect_failures_example_cloud() -> None:
         "invalid_value_of_i_5",
     ]
     with run_app_in_cloud(os.path.join(_PATH_APPS, "collect_failures")) as (
+        _,
         _,
         fetch_logs,
         _,
diff --git a/tests/integrations_app/local/test_custom_work_dependencies.py b/tests/integrations_app/local/test_custom_work_dependencies.py
index f0fe338a8aefd..33400d7c96e81 100644
--- a/tests/integrations_app/local/test_custom_work_dependencies.py
+++ b/tests/integrations_app/local/test_custom_work_dependencies.py
@@ -13,7 +13,7 @@ def test_custom_work_dependencies_example_cloud() -> None:
     with run_app_in_cloud(
         os.path.join(_PATH_APPS, "custom_work_dependencies"),
         app_name="app.py",
-    ) as (_, fetch_logs, _):
+    ) as (_, _, fetch_logs, _):
         has_logs = False
         while not has_logs:
             for log in fetch_logs(["flow"]):
diff --git a/tests/integrations_app/local/test_idle_timeout.py b/tests/integrations_app/local/test_idle_timeout.py
index 35a6d42861725..ba0daf2dc379c 100644
--- a/tests/integrations_app/local/test_idle_timeout.py
+++ b/tests/integrations_app/local/test_idle_timeout.py
@@ -10,6 +10,7 @@
 @pytest.mark.cloud
 def test_idle_timeout_example_cloud() -> None:
     with run_app_in_cloud(os.path.join(_PATH_APPS, "idle_timeout")) as (
+        _,
         _,
         fetch_logs,
         _,
diff --git a/tests/integrations_app/public/test_app_dag.py b/tests/integrations_app/public/test_app_dag.py
index eac1d99939fad..4c1ba3348b2f1 100644
--- a/tests/integrations_app/public/test_app_dag.py
+++ b/tests/integrations_app/public/test_app_dag.py
@@ -9,7 +9,7 @@
 
 @pytest.mark.cloud
 def test_app_dag_example_cloud() -> None:
-    with run_app_in_cloud(os.path.join(_PATH_EXAMPLES, "app_dag")) as (_, fetch_logs, _):
+    with run_app_in_cloud(os.path.join(_PATH_EXAMPLES, "app_dag")) as (_, _, fetch_logs, _):
 
         launch_log, finish_log = False, False
         while not (launch_log and finish_log):
diff --git a/tests/integrations_app/public/test_boring_app.py b/tests/integrations_app/public/test_boring_app.py
index 5ec2b170a7df5..49273918098bb 100644
--- a/tests/integrations_app/public/test_boring_app.py
+++ b/tests/integrations_app/public/test_boring_app.py
@@ -11,6 +11,7 @@
 @pytest.mark.cloud
 def test_boring_app_example_cloud() -> None:
     with run_app_in_cloud(os.path.join(_PATH_EXAMPLES, "app_boring"), app_name="app_dynamic.py", debug=True) as (
+        _,
         view_page,
         _,
         name,
diff --git a/tests/integrations_app/public/test_commands_and_api.py b/tests/integrations_app/public/test_commands_and_api.py
index 2c9c8cc7d77bb..e7288cc125780 100644
--- a/tests/integrations_app/public/test_commands_and_api.py
+++ b/tests/integrations_app/public/test_commands_and_api.py
@@ -12,6 +12,7 @@
 @pytest.mark.cloud
 def test_commands_and_api_example_cloud() -> None:
     with run_app_in_cloud(os.path.join(_PATH_EXAMPLES, "app_commands_and_api")) as (
+        _,
         view_page,
         fetch_logs,
         app_name,
diff --git a/tests/integrations_app/public/test_drive.py b/tests/integrations_app/public/test_drive.py
index 579b1587a341f..757cc66ec5d95 100644
--- a/tests/integrations_app/public/test_drive.py
+++ b/tests/integrations_app/public/test_drive.py
@@ -10,6 +10,7 @@
 @pytest.mark.cloud
 def test_drive_example_cloud() -> None:
     with run_app_in_cloud(os.path.join(_PATH_EXAMPLES, "app_drive")) as (
+        _,
         _,
         fetch_logs,
         _,
diff --git a/tests/integrations_app/public/test_installation_commands_app.py b/tests/integrations_app/public/test_installation_commands_app.py
index 71d6ac7343669..7dbdf932ac650 100644
--- a/tests/integrations_app/public/test_installation_commands_app.py
+++ b/tests/integrations_app/public/test_installation_commands_app.py
@@ -14,7 +14,7 @@ def test_installation_commands_app_example_cloud() -> None:
         app_name="app.py",
         extra_args=["--setup"],
         debug=True,
-    ) as (_, fetch_logs, _):
+    ) as (_, _, fetch_logs, _):
         has_logs = False
         while not has_logs:
             for log in fetch_logs(["work"]):
diff --git a/tests/integrations_app/public/test_payload.py b/tests/integrations_app/public/test_payload.py
index 32daf71d609e3..e4b85c0f180c5 100644
--- a/tests/integrations_app/public/test_payload.py
+++ b/tests/integrations_app/public/test_payload.py
@@ -9,7 +9,7 @@
 
 @pytest.mark.cloud
 def test_payload_example_cloud() -> None:
-    with run_app_in_cloud(os.path.join(_PATH_EXAMPLES, "app_payload")) as (_, fetch_logs, _):
+    with run_app_in_cloud(os.path.join(_PATH_EXAMPLES, "app_payload")) as (_, _, fetch_logs, _):
 
         has_logs = False
         while not has_logs:
diff --git a/tests/integrations_app/public/test_quick_start.py b/tests/integrations_app/public/test_quick_start.py
index 4c59a516248f0..29cae720278bb 100644
--- a/tests/integrations_app/public/test_quick_start.py
+++ b/tests/integrations_app/public/test_quick_start.py
@@ -51,7 +51,7 @@ def test_quick_start_example(caplog, monkeypatch):
 
 @pytest.mark.cloud
 def test_quick_start_example_cloud() -> None:
-    with run_app_in_cloud(os.path.join(_PATH_EXAMPLES, "lightning-quick-start")) as (view_page, _, _):
+    with run_app_in_cloud(os.path.join(_PATH_EXAMPLES, "lightning-quick-start")) as (_, view_page, _, _):
 
         def click_gradio_demo(*_, **__):
             button = view_page.locator('button:has-text("Interactive demo")')
diff --git a/tests/integrations_app/public/test_template_react_ui.py b/tests/integrations_app/public/test_template_react_ui.py
index fb968dace44bc..5cb07510f24bc 100644
--- a/tests/integrations_app/public/test_template_react_ui.py
+++ b/tests/integrations_app/public/test_template_react_ui.py
@@ -11,6 +11,7 @@
 def test_template_react_ui_example_cloud() -> None:
     """This test ensures streamlit works in the cloud by clicking a button and checking the logs."""
     with run_app_in_cloud(os.path.join(_PATH_EXAMPLES, "app_template_react_ui")) as (
+        _,
         view_page,
         fetch_logs,
         _,
diff --git a/tests/integrations_app/public/test_template_streamlit_ui.py b/tests/integrations_app/public/test_template_streamlit_ui.py
index f3a03622d1498..6ca7cd9fd7535 100644
--- a/tests/integrations_app/public/test_template_streamlit_ui.py
+++ b/tests/integrations_app/public/test_template_streamlit_ui.py
@@ -11,6 +11,7 @@
 def test_template_streamlit_ui_example_cloud() -> None:
     """This test ensures streamlit works in the cloud by clicking a button and checking the logs."""
     with run_app_in_cloud(os.path.join(_PATH_EXAMPLES, "app_template_streamlit_ui")) as (
+        _,
         view_page,
         fetch_logs,
         _,
diff --git a/tests/integrations_app/public/test_v0_app.py b/tests/integrations_app/public/test_v0_app.py
index f36c584a5df93..5cc011d15b427 100644
--- a/tests/integrations_app/public/test_v0_app.py
+++ b/tests/integrations_app/public/test_v0_app.py
@@ -67,13 +67,14 @@ def test_v0_app_example_byoc_cloud() -> None:
     with run_app_in_cloud(
         os.path.join(_PATH_EXAMPLES, "app_v0"),
         extra_args=["--cluster-id", os.environ.get("LIGHTNING_BYOC_CLUSTER_ID")],
-    ) as (view_page, fetch_logs, app_name):
+    ) as (_, view_page, fetch_logs, app_name):
         run_v0_app(fetch_logs, view_page)
 
 
 @pytest.mark.cloud
 def test_v0_app_example_cloud() -> None:
     with run_app_in_cloud(os.path.join(_PATH_EXAMPLES, "app_v0")) as (
+        _,
         view_page,
         fetch_logs,
         _,

From 960298ef8909439de2dddf0b193018051f946b68 Mon Sep 17 00:00:00 2001
From: Ethan Harris <ethanwharris@gmail.com>
Date: Wed, 22 Feb 2023 13:48:31 +0000
Subject: [PATCH 14/14] Update tests/integrations_app/flagship/test_flashy.py

---
 tests/integrations_app/flagship/test_flashy.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/integrations_app/flagship/test_flashy.py b/tests/integrations_app/flagship/test_flashy.py
index f680d5d86e371..8a593deeb5282 100644
--- a/tests/integrations_app/flagship/test_flashy.py
+++ b/tests/integrations_app/flagship/test_flashy.py
@@ -73,6 +73,6 @@ def validate_app_functionalities(app_page: "Page") -> None:
 
 @pytest.mark.cloud
 def test_app_cloud() -> None:
-    with run_app_in_cloud(_PATH_INTEGRATIONS_DIR) as (_, view_page, fetch_logs, _):
+    with run_app_in_cloud(_PATH_INTEGRATIONS_DIR) as (admin_page, view_page, fetch_logs, _):
 
         validate_app_functionalities(view_page)