Add CI app cloud e2e

.actions/setup_tools.py

@@ -14,7 +14,12 @@
 import glob
 import logging
 import os
+import pathlib
 import re
+import shutil
+import tarfile
+import tempfile
+import urllib.request
 from importlib.util import module_from_spec, spec_from_file_location
 from itertools import groupby
 from types import ModuleType
@@ -23,6 +28,9 @@
 _PROJECT_ROOT = os.path.dirname(os.path.dirname(__file__))
 _PACKAGE_MAPPING = {"pytorch": "pytorch_lightning", "app": "lightning_app"}
 
+# TODO: remove this once lightning-ui package is ready as a dependency
+_LIGHTNING_FRONTEND_RELEASE_URL = "https://storage.googleapis.com/grid-packages/lightning-ui/v0.0.0/build.tar.gz"
+
 
 def _load_py_module(name: str, location: str) -> ModuleType:
     spec = spec_from_file_location(name, location)
@@ -317,3 +325,26 @@ class implementations by cross-imports to the true package.
         os.makedirs(os.path.dirname(new_file), exist_ok=True)
         with open(new_file, "w", encoding="utf-8") as fp:
             fp.writelines(lines)
+
+
+def _download_frontend(root: str = _PROJECT_ROOT):
+    """Downloads an archive file for a specific release of the Lightning frontend and extracts it to the correct
+    directory."""
+
+    try:
+        build_dir = "build"
+        frontend_dir = pathlib.Path(root, "src", "lightning_app", "ui")
+        download_dir = tempfile.mkdtemp()
+
+        shutil.rmtree(frontend_dir, ignore_errors=True)
+        response = urllib.request.urlopen(_LIGHTNING_FRONTEND_RELEASE_URL)
+
+        file = tarfile.open(fileobj=response, mode="r|gz")
+        file.extractall(path=download_dir)
+
+        shutil.move(os.path.join(download_dir, build_dir), frontend_dir)
+        print("The Lightning UI has successfully been downloaded!")
+
+    # If installing from source without internet connection, we don't want to break the installation
+    except Exception:
+        print("The Lightning UI downloading has failed!")

.github/file-filters.yml

@@ -0,0 +1,9 @@
+# This file contains filters to be used in the CI to detect file changes and run the required CI jobs.
+
+app_examples:
+  - "src/lightning_app/**"
+  - "tests/tests_app_examples/**"
+  - "requirements/app/**"
+  - "examples/app_*"
+  - "setup.py"
+  - "src/pytorch_lightning/__version__.py"

.github/workflows/ci-app_cloud_e2e_test.yml

@@ -0,0 +1,186 @@
+name: cloud-testing
+
+# Used to run the e2e tests on lightning.ai
+
+on: # Trigger the workflow on push or pull request, but only for the master branch
+  push:
+    branches:
+      - "master"
+  pull_request:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref }}
+  cancel-in-progress: ${{ github.ref != 'refs/heads/master' }}
+
+jobs:
+  # This is job should once only once per PR to detect file changes so run required jobs.
+  # see .github/file-filters.yml to define file filters and run the jobs based on the output of each filter.
+  # More info: https://github.com/marketplace/actions/paths-changes-filter
+
+  changes:
+    runs-on: ubuntu-latest
+    # Set job outputs to the values from filter step
+    outputs:
+      app_examples: ${{ steps.filter.outputs.app_examples }}
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v2
+        with:
+          python-version: "3.8"
+
+      - uses: dorny/paths-filter@v2
+        id: filter
+        with:
+          filters: .github/file-filters.yml
+
+  cloud-test:
+    name: Cloud Test
+    needs: changes
+    if: ${{ needs.changes.outputs.app_examples == 'true' }}
+    runs-on: ubuntu-20.04
+    strategy:
+      fail-fast: false
+      matrix:
+        app_name:
+          - v0_app
+          - boring_app
+#          - quick_start  # TODO: fix this
+          - template_streamlit_ui
+          - template_react_ui
+          - template_jupyterlab
+          - idle_timeout
+          - collect_failures
+          - custom_work_dependencies
+          - drive
+          - payload
+    timeout-minutes: 35
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v2
+        with:
+          python-version: "3.8"
+
+      - name: Get PR ID
+        id: PR
+        run: |
+          if [ -z ${{github.event.number}} ]; then
+            echo "::set-output name=ID::$(date +%s)"
+          else
+            echo "::set-output name=ID::${{github.event.number}}"
+          fi
+
+# TODO: Enable cache
+#      - name: Cache virtualenv
+#        id: cache-venv
+#        uses: actions/cache@v2
+#        with:
+#          path: ./.venv/
+#          key: ${{ runner.os }}-pip-${{ matrix.app_name }}-${{ hashFiles('requirements/app/base.txt', 'requirements/app/*.txt', 'src/lightning_app/__version__.py') }}
+#          restore-keys: ${{ runner.os }}-venv-${{ matrix.app_name }}-
+
+      - name: Install dependencies
+        shell: bash
+        run: |
+          pip --version
+          python -m pip install -r requirements/app/devel.txt --no-cache --quiet --find-links https://download.pytorch.org/whl/cpu/torch_stable.html
+#        if: steps.cache-venv.outputs.cache-hit != 'true'  # TODO: Enable cache
+
+      - name: Cache Playwright dependencies
+        id: playwright-cache
+        uses: actions/cache@v2
+        with:
+          path: ~/.cache/ms-playwright
+          key: ${{ runner.os }}-playwright-${{ matrix.app_name }}-${{ hashFiles('requirements/app/base.txt', 'requirements/app/*.txt', 'src/lightning_app/__version__.py') }}
+          restore-keys: ${{ runner.os }}-playwright-${{ matrix.app_name }}-
+
+      - name: Install Playwright system dependencies
+        shell: bash
+        run: |
+          python -m pip install playwright
+          python -m playwright install --with-deps
+#        if: steps.playwright-cache.outputs.cache-hit != 'true'  # TODO: Enable cache
+
+      - name: Install lightning
+        run: |
+          pip install -e .
+        shell: bash
+
+      - name: Lightning Install quick-start
+        if: ${{ (matrix.app_name == 'quick_start') }}
+        shell: bash
+        run: |
+          python -m lightning install app lightning/quick-start -y
+
+      - name: Clone Template React UI Repo
+        uses: actions/checkout@v3
+        with:
+          repository: Lightning-AI/lightning-template-react
+          token: ${{ secrets.PAT_GHOST }}
+          ref: 'master'
+          path: examples/app_template_react_ui
+
+      - name: Clone Template Jupyter Lab Repo
+        uses: actions/checkout@v3
+        with:
+          repository: Lightning-AI/lightning-template-jupyterlab
+          token: ${{ secrets.PAT_GHOST }}
+          ref: 'master'
+          path: examples/app_template_jupyterlab
+
+      - name: Copy Template Jupyter Lab Repo tests
+        shell: bash
+        run: cp examples/app_template_jupyterlab/tests/test_template_jupyterlab.py tests/tests_app_examples/test_template_jupyterlab.py
+
+      - name: List pip dependency
+        shell: bash
+        run: |
+          pip list
+
+      - name: Run the tests
+        env:
+          LAI_USER: ${{ secrets.LAI_USER }}
+          LAI_PASS: ${{ secrets.LAI_PASS }}
+          LIGHTNING_USER_ID: ${{ secrets.LIGHTNING_USER_ID }}
+          LIGHTNING_API_KEY: ${{ secrets.LIGHTNING_API_KEY }}
+          LIGHTNING_USERNAME: ${{ secrets.LIGHTNING_USERNAME }}
+          LIGHTNING_CLOUD_URL: ${{ secrets.LIGHTNING_CLOUD_URL }}
+          CLOUD: "1"
+          VIDEO_LOCATION: ./artifacts/videos
+          PR_NUMBER: ${{ steps.PR.outputs.ID }}
+          TEST_APP_NAME: ${{ matrix.app_name }}
+          HAR_LOCATION: ./artifacts/hars
+          SLOW_MO: 50
+        shell: bash
+        run: |
+          mkdir -p ${VIDEO_LOCATION}
+          HEADLESS=1 python -m pytest tests/tests_app_examples/test_${{ matrix.app_name }}.py::test_${{ matrix.app_name }}_example_cloud --timeout=900 --capture=no -v --color=yes
+          # Delete the artifacts if successful
+          rm -r ${VIDEO_LOCATION}/${{ matrix.app_name }}
+
+      - uses: actions/upload-artifact@v2
+
+        if: ${{ always() }}
+        with:
+          name: test-artifacts
+          path: ./artifacts/videos
+
+      - name: Clean Previous Apps
+        if: ${{ always() }}
+        env:
+          LAI_USER: ${{ secrets.LAI_USER }}
+          LAI_PASS: ${{ secrets.LAI_PASS }}
+          LIGHTNING_USER_ID: ${{ secrets.LIGHTNING_USER_ID }}
+          LIGHTNING_API_KEY: ${{ secrets.LIGHTNING_API_KEY }}
+          LIGHTNING_USERNAME: ${{ secrets.LIGHTNING_USERNAME }}
+          LIGHTNING_CLOUD_URL: ${{ secrets.LIGHTNING_CLOUD_URL }}
+          PR_NUMBER: ${{ steps.PR.outputs.ID }}
+          TEST_APP_NAME: ${{ matrix.app_name }}
+          GRID_USER_ID: ${{ secrets.LIGHTNING_USER_ID }}
+          GRID_USER_KEY: ${{ secrets.LIGHTNING_API_KEY }}
+          GRID_URL: ${{ secrets.LIGHTNING_CLOUD_URL }}
+          _GRID_USERNAME: ${{ secrets.LIGHTNING_USERNAME }}
+        shell: bash
+        run: |
+          time python -c "from lightning.app import testing; testing.delete_cloud_lightning_apps()"

examples/app_template_streamlit_ui/app.py

@@ -0,0 +1,48 @@
+import logging
+
+from lightning_app import LightningApp, LightningFlow
+from lightning_app.frontend import StreamlitFrontend
+from lightning_app.utilities.state import AppState
+
+logger = logging.getLogger(__name__)
+
+
+class StreamlitUI(LightningFlow):
+    def __init__(self):
+        super().__init__()
+        self.message_to_print = "Hello World!"
+        self.should_print = False
+
+    def configure_layout(self):
+        return StreamlitFrontend(render_fn=render_fn)
+
+
+def render_fn(state: AppState):
+    import streamlit as st
+
+    should_print = st.button("Should print to the terminal ?")
+
+    if should_print:
+        state.should_print = not state.should_print
+
+    st.write("Currently printing." if state.should_print else "Currently waiting to print.")
+
+
+class HelloWorld(LightningFlow):
+    def __init__(self):
+        super().__init__()
+        self.counter = 0
+        self.streamlit_ui = StreamlitUI()
+
+    def run(self):
+        self.streamlit_ui.run()
+        if self.streamlit_ui.should_print:
+            logger.info(f"{self.counter}: {self.streamlit_ui.message_to_print}")
+            self.counter += 1
+            self.streamlit_ui.should_print = False
+
+    def configure_layout(self):
+        return [{"name": "StreamLitUI", "content": self.streamlit_ui}]
+
+
+app = LightningApp(HelloWorld())

examples/app_template_streamlit_ui/requirements.txt

@@ -0,0 +1 @@
+streamlit

src/lightning/__setup__.py

@@ -40,6 +40,9 @@ def _adjust_manifest(**kwargs: Any) -> None:
             "recursive-include src *.md" + os.linesep,
             "recursive-include requirements *.txt" + os.linesep,
         ]
+
+    # TODO: remove this once lightning-ui package is ready as a dependency
+    lines += ["recursive-include src/lightning_app/ui *" + os.linesep]
     with open(manifest_path, "w") as fp:
         fp.writelines(lines)
 
@@ -64,7 +67,11 @@ def _setup_args(**kwargs: Any) -> Dict[str, Any]:
             if os.path.isdir(d)
         ]
         _requires = list(chain(*_requires))
-    # todo: consider invaliding some additional arguments from packages, for example if include data or safe to zip
+    # TODO: consider invaliding some additional arguments from packages, for example if include data or safe to zip
+
+    # TODO: remove this once lightning-ui package is ready as a dependency
+    _setup_tools._download_frontend(_PROJECT_ROOT)
+
     return dict(
         name="lightning",
         version=_version.version,  # todo: consider adding branch for installation from source

src/lightning_app/__setup__.py

@@ -51,6 +51,10 @@ def _adjust_manifest(**__: Any) -> None:
         "recursive-include src/lightning_app *.md" + os.linesep,
         "recursive-include requirements/app *.txt" + os.linesep,
     ]
+
+    # TODO: remove this once lightning-ui package is ready as a dependency
+    lines += ["recursive-include src/lightning_app/ui *" + os.linesep]
+
     with open(manifest_path, "w") as fp:
         fp.writelines(lines)
 
@@ -63,7 +67,10 @@ def _setup_args(**__: Any) -> Dict[str, Any]:
     _long_description = _setup_tools.load_readme_description(
         _PACKAGE_ROOT, homepage=_about.__homepage__, version=_version.version
     )
-    # TODO: at this point we need to download the UI to the package
+
+    # TODO: remove this once lightning-ui package is ready as a dependency
+    _setup_tools._download_frontend(_PROJECT_ROOT)
+
     return dict(
         name="lightning-app",
         version=_version.version,  # todo: consider using date version + branch for installation from source

src/lightning_app/testing/__init__.py

@@ -1,3 +1,8 @@
-from lightning_app.testing.testing import application_testing, LightningTestApp, run_work_isolated
+from lightning_app.testing.testing import (
+    application_testing,
+    delete_cloud_lightning_apps,
+    LightningTestApp,
+    run_work_isolated,
+)
 
-__all__ = ["application_testing", "run_work_isolated", "LightningTestApp"]
+__all__ = ["application_testing", "run_work_isolated", "LightningTestApp", "delete_cloud_lightning_apps"]