Skip to content

(4/n) Support 2D Parallelism - Loading optimizer states correctly #25262

(4/n) Support 2D Parallelism - Loading optimizer states correctly

(4/n) Support 2D Parallelism - Loading optimizer states correctly #25262

Workflow file for this run

name: Package
# see: https://help.github.com/en/actions/reference/events-that-trigger-workflows
on:
push:
branches: [master, "release/*"]
pull_request:
branches: [master, "release/*"]
types: [opened, reopened, ready_for_review, synchronize] # added `ready_for_review` since draft is skipped
paths:
- ".actions/*"
- "requirements/ci.txt"
- ".github/actions/pkg-check/*"
- ".github/actions/pkg-install/*"
- ".github/workflows/_build-packages.yml"
- ".github/workflows/ci-pkg-install.yml"
- "setup.py"
- "src/**"
- "requirements/**"
- "!requirements/docs.txt"
- "!requirements/*/docs.txt"
- "!*.md"
- "!**/*.md"
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
defaults:
run:
shell: bash
jobs:
build-packages:
uses: ./.github/workflows/_build-packages.yml
with:
artifact-name: dist-packages-${{ github.sha }}
install-pkg:
needs: build-packages
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
os: ["ubuntu-22.04", "macOS-12", "windows-2022"]
pkg-name: ["app", "fabric", "pytorch", "lightning", "notset"]
python-version: ["3.8", "3.11"]
# TODO: add also install from source
include:
- { os: "macOS-12", pkg-name: "lightning", python-version: "3.9", pkg-extra: "app" }
- { os: "macOS-12", pkg-name: "notset", python-version: "3.9", pkg-extra: "app" }
- { os: "ubuntu-22.04", pkg-name: "lightning", python-version: "3.9", pkg-extra: "app" }
- { os: "ubuntu-22.04", pkg-name: "notset", python-version: "3.9", pkg-extra: "app" }
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
- uses: actions/download-artifact@v3
with:
name: dist-packages-${{ github.sha }}
path: dist
- name: Set package dir
run: |
python -c "print('PKG_DIR=' + {'notset': 'lightning'}.get('${{matrix.pkg-name}}', '${{matrix.pkg-name}}'))" >> $GITHUB_ENV
- name: Install package - wheel & archive
uses: ./.github/actions/pkg-install
timeout-minutes: 25
with:
pkg-folder: dist/${{ env.PKG_DIR }}
pkg-name: ${{ matrix.pkg-name }}
pkg-extra: ${{ matrix.pkg-extra }}
- name: Run CLI (via python)
if: ${{ (matrix.pkg-name == 'lightning' || matrix.pkg-name == 'notset') && matrix.pkg-extra == 'app' }}
run: python -m lightning --version
- name: Run CLI (direct bash)
if: |
((matrix.pkg-name == 'lightning' || matrix.pkg-name == 'notset') && matrix.pkg-extra == 'app') ||
matrix.pkg-name == 'app'
run: lightning_app --version
- name: DocTests actions
working-directory: .actions/
run: |
pip install -q pytest -r requirements.txt
python -m pytest assistant.py
- name: Adjust code for standalone
if: contains(fromJSON('["app", "fabric", "pytorch"]'), matrix.pkg-name)
run: |
python .actions/assistant.py copy_replace_imports --source_dir="./src" \
--source_import="lightning.pytorch,lightning.fabric,lightning.app" \
--target_import="pytorch_lightning,lightning_fabric,lightning_app"
- name: Rename src folders
working-directory: src/
run: |
python -c "n = '${{matrix.pkg-name}}' ; n = n if n in ('app', 'fabric', 'pytorch') else '' ; print('PKG_NAME=' + n)" >> $GITHUB_ENV
rm -f ./*/__*.py
rm -f ./**/__*.py
mv lightning lit # rename lightning folder to prevent accidental local imports
- name: drop Secondary doctest
if: ${{ matrix.pkg-name == 'lightning' || matrix.pkg-name == 'notset' }}
working-directory: src/lit
run: |
items=("data" "store" "app")
for item in "${items[@]}"; do
if [[ "$item" == "${{ matrix.pkg-extra }}" ]]; then
echo "Skipping $item"
continue # Skip this iteration
fi
echo "Removing $item"
rm -rf $item
done
- name: Install pytest doctest extension
run: |
pip install -q -r requirements/doctests.txt
pip list
- name: DocTest package
env:
LIGHTING_TESTING: 1 # path for require wrapper
PY_IGNORE_IMPORTMISMATCH: 1
run: python -m pytest src/lit/${PKG_NAME} --ignore-glob="**/cli/*-template/**" --doctest-plus