Merge branch 'master' into bugfix/neg-progress

.actions/setup_tools.py

@@ -22,7 +22,7 @@
 import urllib.request
 from datetime import datetime
 from importlib.util import module_from_spec, spec_from_file_location
-from itertools import groupby
+from itertools import chain, groupby
 from types import ModuleType
 from typing import List
 
@@ -45,7 +45,7 @@ def _load_py_module(name: str, location: str) -> ModuleType:
 def load_requirements(
     path_dir: str, file_name: str = "base.txt", comment_char: str = "#", unfreeze: bool = True
 ) -> List[str]:
-    """Load requirements from a file.
+    """Loading requirements from a file.
 
     >>> path_req = os.path.join(_PROJECT_ROOT, "requirements")
     >>> load_requirements(path_req)  # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
@@ -433,3 +433,33 @@ def _download_frontend(root: str = _PROJECT_ROOT):
     # If installing from source without internet connection, we don't want to break the installation
     except Exception:
         print("The Lightning UI downloading has failed!")
+
+
+def _adjust_require_versions(source_dir: str = "src", req_dir: str = "requirements") -> None:
+    """Parse the base requirements and append  as version adjustments if needed `pkg>=X1.Y1.Z1,==X2.Y2.*`."""
+    reqs = load_requirements(req_dir, file_name="base.txt")
+    for i, req in enumerate(reqs):
+        pkg_name = req[: min(req.index(c) for c in ">=" if c in req)]
+        ver_ = parse_version_from_file(os.path.join(source_dir, pkg_name))
+        if not ver_:
+            continue
+        ver2 = ".".join(ver_.split(".")[:2] + ["*"])
+        reqs[i] = f"{req}, =={ver2}"
+
+    with open(os.path.join(req_dir, "base.txt"), "w") as fp:
+        fp.writelines([ln + os.linesep for ln in reqs])
+
+
+def _load_aggregate_requirements(req_dir: str = "requirements", freeze_requirements: bool = False) -> None:
+    """Load all base requirements from all particular packages and prune duplicates."""
+    requires = [
+        load_requirements(d, file_name="base.txt", unfreeze=not freeze_requirements)
+        for d in glob.glob(os.path.join(req_dir, "*"))
+        if os.path.isdir(d)
+    ]
+    if not requires:
+        return None
+    # TODO: add some smarter version aggregation per each package
+    requires = list(chain(*requires))
+    with open(os.path.join(req_dir, "base.txt"), "w") as fp:
+        fp.writelines([ln + os.linesep for ln in requires])

.github/actions/pkg-install/action.yml

@@ -28,7 +28,7 @@ runs:
   - name: Install | Uninstall package - archive
     working-directory: ./dist
     run: |
-      pip install *.tar.gz ${PKG_NAME} ${{ inputs.pip-flags }}
+      pip install *.tar.gz ${{ inputs.pip-flags }}
       pip list | grep lightning
       python -c "import ${PKG_NAME} ; print(${PKG_NAME}.__version__)"
       pip uninstall -y ${PKG_NAME}

.github/workflows/ci-app_cloud_e2e_test.yml

@@ -25,7 +25,7 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - name: Set up Python 3.8
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: "3.8"
 
@@ -59,7 +59,7 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - name: Set up Python 3.8
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: "3.8"
 
@@ -75,7 +75,7 @@ jobs:
 # TODO: Enable cache
 #      - name: Cache virtualenv
 #        id: cache-venv
-#        uses: actions/cache@v2
+#        uses: actions/cache@v3
 #        with:
 #          path: ./.venv/
 #          key: ${{ runner.os }}-pip-${{ matrix.app_name }}-${{ hashFiles('requirements/app/base.txt', 'requirements/app/*.txt', 'src/lightning_app/__version__.py') }}
@@ -90,7 +90,7 @@ jobs:
 
       - name: Cache Playwright dependencies
         id: playwright-cache
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         with:
           path: ~/.cache/ms-playwright
           key: ${{ runner.os }}-playwright-${{ matrix.app_name }}-${{ hashFiles('requirements/app/base.txt', 'requirements/app/*.txt', 'src/lightning_app/__version__.py') }}

.github/workflows/ci-app_examples.yml

@@ -27,7 +27,7 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
 

.github/workflows/ci-app_tests.yml

@@ -31,7 +31,7 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
 

.github/workflows/ci-pytorch_test-full.yml

@@ -55,7 +55,7 @@ jobs:
 
     - name: Set up Python ${{ matrix.python-version }}
       if: ${{ (steps.skip.outputs.continue == '1') }}
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python-version }}
 

.github/workflows/ci-pytorch_test-slow.yml

@@ -48,7 +48,7 @@ jobs:
             echo "::set-output name=continue::1"
         fi
 
-    - uses: actions/setup-python@v2
+    - uses: actions/setup-python@v4
       if: ${{ (steps.skip.outputs.continue == '1') }}
       with:
         python-version: ${{ matrix.python-version }}

.github/workflows/ci_pkg-install.yml

@@ -39,7 +39,7 @@ jobs:
 
     steps:
       - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
+      - uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
 
@@ -73,7 +73,7 @@ jobs:
 
     steps:
       - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
+      - uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
 
@@ -105,7 +105,7 @@ jobs:
 
     steps:
       - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
+      - uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
 

.github/workflows/code-checks.yml

@@ -16,7 +16,7 @@ jobs:
     steps:
     - uses: actions/checkout@v3
 
-    - uses: actions/setup-python@v3
+    - uses: actions/setup-python@v4
       with:
         python-version: 3.9
 

.github/workflows/docs-checks.yml

@@ -22,7 +22,7 @@ jobs:
       - uses: actions/checkout@v2
         with:
           submodules: true
-      - uses: actions/setup-python@v2
+      - uses: actions/setup-python@v4
         with:
           python-version: 3.9
 
@@ -74,7 +74,7 @@ jobs:
         with:
           submodules: true
           # lfs: true
-      - uses: actions/setup-python@v2
+      - uses: actions/setup-python@v4
         with:
           python-version: 3.9
 

.github/workflows/docs-deploy.yml

@@ -13,7 +13,7 @@ jobs:
         # If you're using actions/checkout@v2 you must set persist-credentials to false in most cases for the deployment to work correctly.
         with:
           persist-credentials: false
-      - uses: actions/setup-python@v2
+      - uses: actions/setup-python@v4
         with:
           python-version: 3.8
 
@@ -31,7 +31,7 @@ jobs:
       # Note: This uses an internal pip API and may not always work
       # https://github.com/actions/cache/blob/master/examples.md#multiple-oss-in-a-workflow
       - name: Cache pip
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         with:
           path: ~/.cache/pip
           key: ${{ runner.os }}-deploy-docs-pip-${{ hashFiles('requirements/app/*.txt') }}

.github/workflows/events-nightly.yml

@@ -16,7 +16,7 @@ jobs:
     steps:
       # does nightly releases from feature branch
       - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
+      - uses: actions/setup-python@v4
         with:
           python-version: 3.9
 

.github/workflows/legacy-checkpoints.yml

@@ -10,7 +10,7 @@ jobs:
     steps:
     - uses: actions/checkout@v2
 
-    - uses: actions/setup-python@v2
+    - uses: actions/setup-python@v4
       with:
         python-version: 3.9
 

.github/workflows/release-pypi.yml

@@ -22,7 +22,7 @@ jobs:
       pull-pkgs: ${{ steps.download.outputs.pkgs }}
     steps:
     - uses: actions/checkout@v2
-    - uses: actions/setup-python@v2
+    - uses: actions/setup-python@v4
       with:
         python-version: 3.9
     - run: |
@@ -65,7 +65,7 @@ jobs:
       with:
         name: dist-packages-${{ github.sha }}
         path: dist
-    - uses: actions/setup-python@v2
+    - uses: actions/setup-python@v4
       with:
         python-version: 3.9
 
@@ -99,7 +99,7 @@ jobs:
       with:
         name: pypi-packages-${{ github.sha }}
         path: pypi
-    - uses: actions/setup-python@v2
+    - uses: actions/setup-python@v4
       with:
         python-version: 3.9
 
@@ -129,7 +129,7 @@ jobs:
         name: pypi-packages-${{ github.sha }}
         path: pypi
     - run: ls -lh pypi/
-    - uses: actions/setup-python@v2
+    - uses: actions/setup-python@v4
       with:
         python-version: 3.9
 
@@ -222,7 +222,7 @@ jobs:
     steps:
     - uses: actions/checkout@v2
 
-    - uses: actions/setup-python@v2
+    - uses: actions/setup-python@v4
       with:
         python-version: 3.9
 

.gitignore

@@ -164,3 +164,4 @@ src/lightning_app/ui/*
 hars*
 artifacts/*
 *docs/examples*
+*docs/source-app/api*

docs/source-app/api_reference/core.rst

@@ -0,0 +1,26 @@
+:orphan:
+
+##################
+lightning_app.core
+##################
+
+.. contents::
+    :depth: 1
+    :local:
+    :backlinks: top
+
+.. currentmodule:: lightning_app.core
+
+Core APIs
+___________________
+
+.. autosummary::
+    :toctree: api/
+    :nosignatures:
+    :template: classtemplate.rst
+
+    LightningApp
+    LightningFlow
+    LightningWork
+
+Learn more about :ref:`Lightning Core <core_api>`.

docs/source-app/api_reference/api_references.rst → docs/source-app/api_references.rst

@@ -5,12 +5,12 @@ Lightning App - API References
 ##############################
 
 Core
-----
+____
 
 .. currentmodule:: lightning_app.core
 
 .. autosummary::
-    :toctree: api
+    :toctree: api/
     :nosignatures:
     :template: classtemplate_no_index.rst
 
@@ -32,10 +32,10 @@ ___________________
     :nosignatures:
     :template: classtemplate_no_index.rst
 
-    ~serve.serve.ModelInferenceAPI
     ~python.popen.PopenPythonScript
-    ~serve.gradio.ServeGradio
     ~python.tracer.TracerPythonScript
+    ~serve.gradio.ServeGradio
+    ~serve.serve.ModelInferenceAPI
 
 ----
 
@@ -67,8 +67,8 @@ _______
     :nosignatures:
     :template: classtemplate_no_index.rst
 
-    ~drive.Drive
     ~path.Path
+    ~drive.Drive
     ~payload.Payload
 
 Learn more about :ref:`Storage <storage>`.
@@ -86,5 +86,5 @@ _______
     :template: classtemplate_no_index.rst
 
     ~cloud.CloudRuntime
-    ~multiprocess.MultiProcessRuntime
     ~singleprocess.SingleProcessRuntime
+    ~multiprocess.MultiProcessRuntime

docs/source-app/examples/file_server/app.py

@@ -10,7 +10,13 @@
 
 
 class FileServer(L.LightningWork):
-    def __init__(self, drive: Drive, base_dir: str = "file_server", chunk_size=10240, **kwargs):
+    def __init__(
+        self,
+        drive: Drive,
+        base_dir: str = "file_server",
+        chunk_size=10240,
+        **kwargs
+    ):
         """This component uploads, downloads files to your application.
 
         Arguments:
@@ -48,7 +54,9 @@ def upload_file(self, file):
         filename = file.filename
         uploaded_file = self.get_random_filename()
         meta_file = uploaded_file + ".meta"
-        self.uploaded_files[filename] = {"progress": (0, None), "done": False}
+        self.uploaded_files[filename] = {
+            "progress": (0, None), "done": False
+        }
 
         # 2: Create a stream and write bytes of
         # the file to the disk under `uploaded_file` path.
@@ -155,6 +163,7 @@ def alive(self):
 
 
 class TestFileServer(LightningWork):
+
     def __init__(self, drive: Drive):
         super().__init__(cache_calls=True)
         self.drive = drive
@@ -164,7 +173,10 @@ def run(self, file_server_url: str, first=True):
             with open("test.txt", "w") as f:
                 f.write("Some text.")
 
-            response = requests.post(file_server_url + "/upload_file/", files={"file": open("test.txt", "rb")})
+            response = requests.post(
+                file_server_url + "/upload_file/",
+                files={'file': open("test.txt", 'rb')}
+            )
             assert response.status_code == 200
         else:
             response = requests.get(file_server_url)
@@ -176,6 +188,7 @@ def run(self, file_server_url: str, first=True):
 
 
 class Flow(LightningFlow):
+
     def __init__(self):
         super().__init__()
         # 1: Create a drive to share data between works