diff --git a/.actions/setup_tools.py b/.actions/setup_tools.py index 2790820a5a221..3a105f508fd45 100644 --- a/.actions/setup_tools.py +++ b/.actions/setup_tools.py @@ -14,7 +14,12 @@ import glob import logging import os +import pathlib import re +import shutil +import tarfile +import tempfile +import urllib.request from importlib.util import module_from_spec, spec_from_file_location from itertools import groupby from types import ModuleType @@ -23,6 +28,9 @@ _PROJECT_ROOT = os.path.dirname(os.path.dirname(__file__)) _PACKAGE_MAPPING = {"pytorch": "pytorch_lightning", "app": "lightning_app"} +# TODO: remove this once lightning-ui package is ready as a dependency +_LIGHTNING_FRONTEND_RELEASE_URL = "https://storage.googleapis.com/grid-packages/lightning-ui/v0.0.0/build.tar.gz" + def _load_py_module(name: str, location: str) -> ModuleType: spec = spec_from_file_location(name, location) @@ -317,3 +325,26 @@ class implementations by cross-imports to the true package. os.makedirs(os.path.dirname(new_file), exist_ok=True) with open(new_file, "w", encoding="utf-8") as fp: fp.writelines(lines) + + +def _download_frontend(root: str = _PROJECT_ROOT): + """Downloads an archive file for a specific release of the Lightning frontend and extracts it to the correct + directory.""" + + try: + build_dir = "build" + frontend_dir = pathlib.Path(root, "src", "lightning_app", "ui") + download_dir = tempfile.mkdtemp() + + shutil.rmtree(frontend_dir, ignore_errors=True) + response = urllib.request.urlopen(_LIGHTNING_FRONTEND_RELEASE_URL) + + file = tarfile.open(fileobj=response, mode="r|gz") + file.extractall(path=download_dir) + + shutil.move(os.path.join(download_dir, build_dir), frontend_dir) + print("The Lightning UI has successfully been downloaded!") + + # If installing from source without internet connection, we don't want to break the installation + except Exception: + print("The Lightning UI downloading has failed!") diff --git a/.github/file-filters.yml b/.github/file-filters.yml new file mode 100644 index 0000000000000..e621cd83881e4 --- /dev/null +++ b/.github/file-filters.yml @@ -0,0 +1,9 @@ +# This file contains filters to be used in the CI to detect file changes and run the required CI jobs. + +app_examples: + - "src/lightning_app/**" + - "tests/tests_app_examples/**" + - "requirements/app/**" + - "examples/app_*" + - "setup.py" + - "src/pytorch_lightning/__version__.py" diff --git a/.github/workflows/ci-app_cloud_e2e_test.yml b/.github/workflows/ci-app_cloud_e2e_test.yml new file mode 100644 index 0000000000000..341c2dc20e83d --- /dev/null +++ b/.github/workflows/ci-app_cloud_e2e_test.yml @@ -0,0 +1,186 @@ +name: cloud-testing + +# Used to run the e2e tests on lightning.ai + +on: # Trigger the workflow on push or pull request, but only for the master branch + push: + branches: + - "master" + pull_request: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/master' }} + +jobs: + # This is job should once only once per PR to detect file changes so run required jobs. + # see .github/file-filters.yml to define file filters and run the jobs based on the output of each filter. + # More info: https://github.com/marketplace/actions/paths-changes-filter + + changes: + runs-on: ubuntu-latest + # Set job outputs to the values from filter step + outputs: + app_examples: ${{ steps.filter.outputs.app_examples }} + steps: + - uses: actions/checkout@v2 + - name: Set up Python 3.8 + uses: actions/setup-python@v2 + with: + python-version: "3.8" + + - uses: dorny/paths-filter@v2 + id: filter + with: + filters: .github/file-filters.yml + + cloud-test: + name: Cloud Test + needs: changes + if: ${{ needs.changes.outputs.app_examples == 'true' }} + runs-on: ubuntu-20.04 + strategy: + fail-fast: false + matrix: + app_name: + - v0_app + - boring_app +# - quick_start # TODO: fix this + - template_streamlit_ui + - template_react_ui + - template_jupyterlab + - idle_timeout + - collect_failures + - custom_work_dependencies + - drive + - payload + timeout-minutes: 35 + steps: + - uses: actions/checkout@v2 + - name: Set up Python 3.8 + uses: actions/setup-python@v2 + with: + python-version: "3.8" + + - name: Get PR ID + id: PR + run: | + if [ -z ${{github.event.number}} ]; then + echo "::set-output name=ID::$(date +%s)" + else + echo "::set-output name=ID::${{github.event.number}}" + fi + +# TODO: Enable cache +# - name: Cache virtualenv +# id: cache-venv +# uses: actions/cache@v2 +# with: +# path: ./.venv/ +# key: ${{ runner.os }}-pip-${{ matrix.app_name }}-${{ hashFiles('requirements/app/base.txt', 'requirements/app/*.txt', 'src/lightning_app/__version__.py') }} +# restore-keys: ${{ runner.os }}-venv-${{ matrix.app_name }}- + + - name: Install dependencies + shell: bash + run: | + pip --version + python -m pip install -r requirements/app/devel.txt --no-cache --quiet --find-links https://download.pytorch.org/whl/cpu/torch_stable.html +# if: steps.cache-venv.outputs.cache-hit != 'true' # TODO: Enable cache + + - name: Cache Playwright dependencies + id: playwright-cache + uses: actions/cache@v2 + with: + path: ~/.cache/ms-playwright + key: ${{ runner.os }}-playwright-${{ matrix.app_name }}-${{ hashFiles('requirements/app/base.txt', 'requirements/app/*.txt', 'src/lightning_app/__version__.py') }} + restore-keys: ${{ runner.os }}-playwright-${{ matrix.app_name }}- + + - name: Install Playwright system dependencies + shell: bash + run: | + python -m pip install playwright + python -m playwright install --with-deps +# if: steps.playwright-cache.outputs.cache-hit != 'true' # TODO: Enable cache + + - name: Install lightning + run: | + pip install -e . + shell: bash + + - name: Lightning Install quick-start + if: ${{ (matrix.app_name == 'quick_start') }} + shell: bash + run: | + python -m lightning install app lightning/quick-start -y + + - name: Clone Template React UI Repo + uses: actions/checkout@v3 + with: + repository: Lightning-AI/lightning-template-react + token: ${{ secrets.PAT_GHOST }} + ref: 'master' + path: examples/app_template_react_ui + + - name: Clone Template Jupyter Lab Repo + uses: actions/checkout@v3 + with: + repository: Lightning-AI/lightning-template-jupyterlab + token: ${{ secrets.PAT_GHOST }} + ref: 'master' + path: examples/app_template_jupyterlab + + - name: Copy Template Jupyter Lab Repo tests + shell: bash + run: cp examples/app_template_jupyterlab/tests/test_template_jupyterlab.py tests/tests_app_examples/test_template_jupyterlab.py + + - name: List pip dependency + shell: bash + run: | + pip list + + - name: Run the tests + env: + LAI_USER: ${{ secrets.LAI_USER }} + LAI_PASS: ${{ secrets.LAI_PASS }} + LIGHTNING_USER_ID: ${{ secrets.LIGHTNING_USER_ID }} + LIGHTNING_API_KEY: ${{ secrets.LIGHTNING_API_KEY }} + LIGHTNING_USERNAME: ${{ secrets.LIGHTNING_USERNAME }} + LIGHTNING_CLOUD_URL: ${{ secrets.LIGHTNING_CLOUD_URL }} + CLOUD: "1" + VIDEO_LOCATION: ./artifacts/videos + PR_NUMBER: ${{ steps.PR.outputs.ID }} + TEST_APP_NAME: ${{ matrix.app_name }} + HAR_LOCATION: ./artifacts/hars + SLOW_MO: 50 + shell: bash + run: | + mkdir -p ${VIDEO_LOCATION} + HEADLESS=1 python -m pytest tests/tests_app_examples/test_${{ matrix.app_name }}.py::test_${{ matrix.app_name }}_example_cloud --timeout=900 --capture=no -v --color=yes + # Delete the artifacts if successful + rm -r ${VIDEO_LOCATION}/${{ matrix.app_name }} + + - uses: actions/upload-artifact@v2 + + if: ${{ always() }} + with: + name: test-artifacts + path: ./artifacts/videos + + - name: Clean Previous Apps + if: ${{ always() }} + env: + LAI_USER: ${{ secrets.LAI_USER }} + LAI_PASS: ${{ secrets.LAI_PASS }} + LIGHTNING_USER_ID: ${{ secrets.LIGHTNING_USER_ID }} + LIGHTNING_API_KEY: ${{ secrets.LIGHTNING_API_KEY }} + LIGHTNING_USERNAME: ${{ secrets.LIGHTNING_USERNAME }} + LIGHTNING_CLOUD_URL: ${{ secrets.LIGHTNING_CLOUD_URL }} + PR_NUMBER: ${{ steps.PR.outputs.ID }} + TEST_APP_NAME: ${{ matrix.app_name }} + GRID_USER_ID: ${{ secrets.LIGHTNING_USER_ID }} + GRID_USER_KEY: ${{ secrets.LIGHTNING_API_KEY }} + GRID_URL: ${{ secrets.LIGHTNING_CLOUD_URL }} + _GRID_USERNAME: ${{ secrets.LIGHTNING_USERNAME }} + shell: bash + run: | + time python -c "from lightning.app import testing; testing.delete_cloud_lightning_apps()" diff --git a/examples/app_template_streamlit_ui/app.py b/examples/app_template_streamlit_ui/app.py new file mode 100644 index 0000000000000..33aa3dd26f700 --- /dev/null +++ b/examples/app_template_streamlit_ui/app.py @@ -0,0 +1,48 @@ +import logging + +from lightning_app import LightningApp, LightningFlow +from lightning_app.frontend import StreamlitFrontend +from lightning_app.utilities.state import AppState + +logger = logging.getLogger(__name__) + + +class StreamlitUI(LightningFlow): + def __init__(self): + super().__init__() + self.message_to_print = "Hello World!" + self.should_print = False + + def configure_layout(self): + return StreamlitFrontend(render_fn=render_fn) + + +def render_fn(state: AppState): + import streamlit as st + + should_print = st.button("Should print to the terminal ?") + + if should_print: + state.should_print = not state.should_print + + st.write("Currently printing." if state.should_print else "Currently waiting to print.") + + +class HelloWorld(LightningFlow): + def __init__(self): + super().__init__() + self.counter = 0 + self.streamlit_ui = StreamlitUI() + + def run(self): + self.streamlit_ui.run() + if self.streamlit_ui.should_print: + logger.info(f"{self.counter}: {self.streamlit_ui.message_to_print}") + self.counter += 1 + self.streamlit_ui.should_print = False + + def configure_layout(self): + return [{"name": "StreamLitUI", "content": self.streamlit_ui}] + + +app = LightningApp(HelloWorld()) diff --git a/examples/app_template_streamlit_ui/requirements.txt b/examples/app_template_streamlit_ui/requirements.txt new file mode 100644 index 0000000000000..12a4706528df6 --- /dev/null +++ b/examples/app_template_streamlit_ui/requirements.txt @@ -0,0 +1 @@ +streamlit diff --git a/src/lightning/__setup__.py b/src/lightning/__setup__.py index 439ece715425a..296e28eca67e7 100644 --- a/src/lightning/__setup__.py +++ b/src/lightning/__setup__.py @@ -40,6 +40,9 @@ def _adjust_manifest(**kwargs: Any) -> None: "recursive-include src *.md" + os.linesep, "recursive-include requirements *.txt" + os.linesep, ] + + # TODO: remove this once lightning-ui package is ready as a dependency + lines += ["recursive-include src/lightning_app/ui *" + os.linesep] with open(manifest_path, "w") as fp: fp.writelines(lines) @@ -64,7 +67,11 @@ def _setup_args(**kwargs: Any) -> Dict[str, Any]: if os.path.isdir(d) ] _requires = list(chain(*_requires)) - # todo: consider invaliding some additional arguments from packages, for example if include data or safe to zip + # TODO: consider invaliding some additional arguments from packages, for example if include data or safe to zip + + # TODO: remove this once lightning-ui package is ready as a dependency + _setup_tools._download_frontend(_PROJECT_ROOT) + return dict( name="lightning", version=_version.version, # todo: consider adding branch for installation from source diff --git a/src/lightning_app/__setup__.py b/src/lightning_app/__setup__.py index 2a738f20c6b7c..0e892fa047ab7 100644 --- a/src/lightning_app/__setup__.py +++ b/src/lightning_app/__setup__.py @@ -51,6 +51,10 @@ def _adjust_manifest(**__: Any) -> None: "recursive-include src/lightning_app *.md" + os.linesep, "recursive-include requirements/app *.txt" + os.linesep, ] + + # TODO: remove this once lightning-ui package is ready as a dependency + lines += ["recursive-include src/lightning_app/ui *" + os.linesep] + with open(manifest_path, "w") as fp: fp.writelines(lines) @@ -63,7 +67,10 @@ def _setup_args(**__: Any) -> Dict[str, Any]: _long_description = _setup_tools.load_readme_description( _PACKAGE_ROOT, homepage=_about.__homepage__, version=_version.version ) - # TODO: at this point we need to download the UI to the package + + # TODO: remove this once lightning-ui package is ready as a dependency + _setup_tools._download_frontend(_PROJECT_ROOT) + return dict( name="lightning-app", version=_version.version, # todo: consider using date version + branch for installation from source diff --git a/src/lightning_app/testing/__init__.py b/src/lightning_app/testing/__init__.py index 0d2fe1ca899b6..e2401f309c8e1 100644 --- a/src/lightning_app/testing/__init__.py +++ b/src/lightning_app/testing/__init__.py @@ -1,3 +1,8 @@ -from lightning_app.testing.testing import application_testing, LightningTestApp, run_work_isolated +from lightning_app.testing.testing import ( + application_testing, + delete_cloud_lightning_apps, + LightningTestApp, + run_work_isolated, +) -__all__ = ["application_testing", "run_work_isolated", "LightningTestApp"] +__all__ = ["application_testing", "run_work_isolated", "LightningTestApp", "delete_cloud_lightning_apps"] diff --git a/src/lightning_app/testing/testing.py b/src/lightning_app/testing/testing.py index e72c6d05ae4b0..9e7c727756ba0 100644 --- a/src/lightning_app/testing/testing.py +++ b/src/lightning_app/testing/testing.py @@ -186,7 +186,9 @@ def run_app_in_cloud(app_folder: str, app_name: str = "app.py") -> Generator: } context = browser.new_context( # Eventually this will need to be deleted - http_credentials=HttpCredentials({"username": os.getenv("LAI_USER"), "password": os.getenv("LAI_PASS")}), + http_credentials=HttpCredentials( + {"username": os.getenv("LAI_USER").strip(), "password": os.getenv("LAI_PASS")} + ), record_video_dir=os.path.join(Config.video_location, TEST_APP_NAME), record_har_path=Config.har_location, ) @@ -329,3 +331,39 @@ def wait_for(page, callback: Callable, *args, **kwargs) -> Any: print(e) pass sleep(2) + + +def delete_cloud_lightning_apps(): + """Cleanup cloud apps that start with the name test-{PR_NUMBER}-{TEST_APP_NAME}. + + PR_NUMBER and TEST_APP_NAME are environment variables. + """ + + client = LightningClient() + + try: + pr_number = int(os.getenv("PR_NUMBER", None)) + except (TypeError, ValueError): + # Failed when the PR is running master or 'PR_NUMBER' isn't defined. + pr_number = "" + + app_name = os.getenv("TEST_APP_NAME", "") + + print(f"deleting apps for pr_number: {pr_number}, app_name: {app_name}") + project = _get_project(client) + list_lightningapps = client.lightningapp_instance_service_list_lightningapp_instances(project.project_id) + + print([lightningapp.name for lightningapp in list_lightningapps.lightningapps]) + + for lightningapp in list_lightningapps.lightningapps: + if pr_number and app_name and not lightningapp.name.startswith(f"test-{pr_number}-{app_name}-"): + continue + print(f"Deleting {lightningapp.name}") + try: + res = client.lightningapp_instance_service_delete_lightningapp_instance( + project_id=project.project_id, + id=lightningapp.id, + ) + assert res == {} + except ApiException as e: + print(f"Failed to delete {lightningapp.name}. Exception {e}") diff --git a/src/lightning_app/utilities/packaging/lightning_utils.py b/src/lightning_app/utilities/packaging/lightning_utils.py index c6bcea035797f..ae26d39ec5bbb 100644 --- a/src/lightning_app/utilities/packaging/lightning_utils.py +++ b/src/lightning_app/utilities/packaging/lightning_utils.py @@ -25,7 +25,7 @@ LIGHTNING_FRONTEND_RELEASE_URL = "https://storage.googleapis.com/grid-packages/lightning-ui/v0.0.0/build.tar.gz" -def download_frontend(root): +def download_frontend(root: str = _PROJECT_ROOT): """Downloads an archive file for a specific release of the Lightning frontend and extracts it to the correct directory.""" build_dir = "build" diff --git a/tests/tests_app_examples/test_quick_start.py b/tests/tests_app_examples/test_quick_start.py index 272fdbb7f5b63..9db693a5dc3d6 100644 --- a/tests/tests_app_examples/test_quick_start.py +++ b/tests/tests_app_examples/test_quick_start.py @@ -24,7 +24,7 @@ def run_once(self): return done -# TODO +# TODO: Investigate why it doesn't work @pytest.mark.skipif(True, reason="test is skipped because CI was blocking all the PRs.") @RunIf(pytorch_lightning=True, skip_windows=True, skip_linux=True) def test_quick_start_example(caplog, monkeypatch): diff --git a/tests/tests_app_examples/test_template_streamlit_ui.py b/tests/tests_app_examples/test_template_streamlit_ui.py index ec18206bf06dd..a8ba93794f2a0 100644 --- a/tests/tests_app_examples/test_template_streamlit_ui.py +++ b/tests/tests_app_examples/test_template_streamlit_ui.py @@ -10,7 +10,7 @@ @pytest.mark.cloud def test_template_streamlit_ui_example_cloud() -> None: """This test ensures streamlit works in the cloud by clicking a button and checking the logs.""" - with run_app_in_cloud(os.path.join(_PROJECT_ROOT, "templates/template_streamlit_ui/")) as ( + with run_app_in_cloud(os.path.join(_PROJECT_ROOT, "examples/app_template_streamlit_ui")) as ( _, view_page, fetch_logs,