CI: clean install & share pkg build

.github/actions/pkg-check/action.yml

@@ -5,7 +5,6 @@ inputs:
   pkg-name:
     description: package name inside lightning.*
     required: true
-    default: ""
   nb-dirs:
     description: nb of packages in the wrap/distribution
     required: false

.github/actions/pkg-install/action.yml

@@ -21,15 +21,15 @@ runs:
   - name: Install package - archive
     working-directory: ./dist
     run: |
-      pip install *.tar.gz ${{ inputs.pip-flags }}
+      pip install ${PKG_IMPORT} --no-binary ${PKG_IMPORT} --force-reinstall ${{ inputs.pip-flags }}
       pip list | grep lightning
       python -c "import ${{ env.PKG_IMPORT }}; print(${{ env.PKG_IMPORT }}.__version__)"
     shell: bash
 
   - name: Install package - wheel
     working-directory: ./dist
     run: |
-      pip install *.whl ${{ inputs.pip-flags }}
+      pip install ${PKG_IMPORT} --only-binary ${PKG_IMPORT} --force-reinstall ${{ inputs.pip-flags }}
       pip list | grep lightning
       python -c "import ${{ env.PKG_IMPORT }}; print(${{ env.PKG_IMPORT }}.__version__)"
     shell: bash

.github/workflows/_build-packages.yml

@@ -0,0 +1,63 @@
+name: Building packages
+
+on:
+  workflow_call:
+    inputs:
+      artifact-name:
+        description: 'Unique name for collecting artifacts'
+        required: true
+        type: string
+      pkg-names:
+        description: 'list package names to be build in json format'
+        required: false
+        type: string
+        default: |
+          ["lightning", "app", "lite", "pytorch"]
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+
+  init:
+    runs-on: ubuntu-20.04
+    steps:
+    - uses: actions/checkout@v3
+    - run: |
+        mkdir dist && touch dist/.placeholder
+    - uses: actions/upload-artifact@v3
+      with:
+        name: ${{ inputs.artifact-name }}
+        path: dist
+
+
+  build-packages:
+    needs: init
+    runs-on: ubuntu-20.04
+    strategy:
+      max-parallel: 1  # run sequential to prevent download/upload collisions
+      matrix:
+        pkg-name: ${{ fromJSON(inputs.pkg-names) }}
+    steps:
+    - uses: actions/checkout@v3
+    - uses: actions/download-artifact@v3
+      with:
+        name: ${{ inputs.artifact-name }}
+        path: pypi
+    - uses: actions/setup-python@v4
+      with:
+        python-version: 3.9
+
+    - run: python -c "print('NB_DIRS=' + str(2 if '${{ matrix.pkg-name }}' == 'pytorch' else 1))" >> $GITHUB_ENV
+    - uses: ./.github/actions/pkg-check
+      with:
+        pkg-name: ${{ matrix.pkg-name }}
+        nb-dirs: ${{ env.NB_DIRS }}
+
+    - run: cp dist/* pypi/
+
+    - uses: actions/upload-artifact@v3
+      with:
+        name: ${{ inputs.artifact-name }}
+        path: pypi

.github/workflows/ci-pkg-install.yml

@@ -9,6 +9,9 @@ on:
     types: [opened, reopened, ready_for_review, synchronize]  # added `ready_for_review` since draft is skipped
     paths:
       - ".actions/**"
+      - ".github/actions/pkg-check/*"
+      - ".github/actions/pkg-install/*"
+      - ".github/workflows/_build-packages.yml"
       - ".github/workflows/ci-pkg-install.yml"
       - "setup.py"
       - "src/**"
@@ -28,7 +31,13 @@ defaults:
 
 jobs:
 
+  build-packages:
+    uses: ./.github/workflows/_build-packages.yml
+    with:
+      artifact-name: dist-packages-${{ github.sha }}
+
   install-pkg:
+    needs: build-packages
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
@@ -38,33 +47,30 @@ jobs:
         python-version: ["3.7" , "3.10"]
     steps:
       - uses: actions/checkout@v3
-
       - uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
-
-      - name: DocTests actions
-        working-directory: .actions/
-        run: |
-          pip install -q pytest
-          python -m pytest setup_tools.py
-
-      - run: python -c "print('NB_DIRS=' + str(2 if '${{ matrix.pkg-name }}' == 'pytorch' else 1))" >> $GITHUB_ENV
-
-      - uses: ./.github/actions/pkg-check
+      - uses: actions/download-artifact@v3
         with:
-          pkg-name: ${{ matrix.pkg-name }}
-          nb-dirs: ${{ env.NB_DIRS }}
+          name: dist-packages-${{ github.sha }}
+          path: dist
 
       - uses: ./.github/actions/pkg-install
         with:
           pkg-name: ${{ matrix.pkg-name }}
+          pip-flags: "-f dist"
 
       - name: Run CLI
         # todo: add testing for `lightning_app`
         if: ${{ matrix.pkg-name == 'lightning' }}
         run: python -m lightning --version
 
+      - name: DocTests actions
+        working-directory: .actions/
+        run: |
+          pip install -q pytest
+          python -m pytest setup_tools.py
+
       - name: DocTest package
         env:
           LIGHTING_TESTING: 1  # path for require wrapper

.github/workflows/release-pypi.yml

@@ -13,47 +13,10 @@ defaults:
 
 jobs:
 
-  init:
-    runs-on: ubuntu-20.04
-    steps:
-    - uses: actions/checkout@v3
-    - run: |
-        mkdir dist && touch dist/.placeholder
-    - uses: actions/upload-artifact@v3
-      with:
-        name: dist-packages-${{ github.sha }}
-        path: dist
-
-
   build-packages:
-    needs: init
-    runs-on: ubuntu-20.04
-    strategy:
-      fail-fast: true
-      max-parallel: 1  # run sequential to prevent download/upload collisions
-      matrix:
-        pkg-name: ["lightning", "app", "lite", "pytorch"]
-    steps:
-    - uses: actions/checkout@v3
-    - uses: actions/download-artifact@v3
-      with:
-        name: dist-packages-${{ github.sha }}
-        path: dist
-    - uses: actions/setup-python@v4
-      with:
-        python-version: 3.9
-    - name: Install dependencies
-      run: pip install -U setuptools wheel
-    - name: Build packages
-      env:
-        PACKAGE_NAME: ${{ matrix.pkg-name }}
-      run: |
-        python setup.py sdist bdist_wheel
-        ls -lh dist/
-    - uses: actions/upload-artifact@v3
-      with:
-        name: dist-packages-${{ github.sha }}
-        path: dist
+    uses: ./.github/workflows/_build-packages.yml
+    with:
+      artifact-name: dist-packages-${{ github.sha }}
 
 
   upload-packages:

src/lightning_app/components/python/popen.py

@@ -41,8 +41,7 @@ def __init__(
         Raises:
             FileNotFoundError: If the provided `script_path` doesn't exists.
 
-        .. doctest::
-
+        Example:
 
             >>> from lightning_app.components.python import PopenPythonScript
             >>> f = open("a.py", "w")

src/lightning_app/components/python/tracer.py

@@ -70,7 +70,7 @@ def __init__(
         This method takes any python globals before executing the script,
         e.g., you can modify classes or function from the script.
 
-        .. doctest::
+        Example:
 
             >>> from lightning_app.components.python import TracerPythonScript
             >>> f = open("a.py", "w")

src/lightning_app/core/app.py

@@ -87,7 +87,7 @@ def __init__(
                 You can learn more about proxy `here <https://www.fortinet.com/resources/cyberglossary/proxy-server>`_.
 
 
-        .. doctest::
+        Example:
 
             >>> from lightning_app import LightningFlow, LightningApp
             >>> from lightning_app.runners import MultiProcessRuntime

src/lightning_app/core/flow.py

@@ -77,7 +77,7 @@ def __init__(self):
         can be distributed (each LightningWork will be run within its own process
         or different arrangements).
 
-        .. doctest::
+        Example:
 
             >>> from lightning_app import LightningFlow
             >>> class RootFlow(LightningFlow):

src/lightning_app/structures/dict.py

@@ -19,7 +19,7 @@ def __init__(self, **kwargs: T):
         :class:`~lightning_app.core.work.LightningWork`
         or :class:`~lightning_app.core.flow.LightningFlow`.
 
-        .. doctest::
+        Example:
 
             >>> from lightning_app import LightningFlow, LightningWork
             >>> from lightning_app.structures import Dict

src/lightning_app/structures/list.py

@@ -19,7 +19,7 @@ def __init__(self, *items: T):
         :class:`~lightning_app.core.work.LightningWork`
         or :class:`~lightning_app.core.flow.LightningFlow`.
 
-        .. doctest::
+        Example:
 
             >>> from lightning_app import LightningFlow, LightningWork
             >>> from lightning_app.structures import List

src/lightning_app/utilities/packaging/build_config.py

@@ -91,7 +91,7 @@ def build_commands(self) -> List[str]:
 
         .. note:: If you provide your own dockerfile, this would be ignored.
 
-        .. doctest::
+        Example:
 
             from dataclasses import dataclass
             from lightning_app import BuildConfig