BLD: Fix self-hosted CI env (#102) #506
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Python CI | |
| on: | |
| push: | |
| branches: | |
| - '*' | |
| pull_request: | |
| types: ['opened', 'reopened', 'synchronize'] | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| lint: | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ "ubuntu-latest" ] | |
| python-version: [ "3.10" ] | |
| steps: | |
| - name: Check out code | |
| uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| submodules: recursive | |
| - name: Set up Python environment | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: "3.10" | |
| - name: flake8 Lint | |
| uses: py-actions/flake8@v2 | |
| with: | |
| path: "python/xoscar" | |
| args: "--config python/setup.cfg" | |
| - name: black | |
| uses: psf/black@stable | |
| with: | |
| src: "python/xoscar" | |
| options: "--check" | |
| - uses: isort/isort-action@master | |
| with: | |
| sortPaths: "python/xoscar" | |
| configuration: "--check-only --diff --sp python/setup.cfg" | |
| - name: mypy | |
| run: pip install mypy && cd python && mypy xoscar | |
| - name: codespell | |
| run: pip install codespell && cd python && codespell xoscar | |
| - name: clang-format | |
| uses: jidicula/clang-format-action@v4.11.0 | |
| with: | |
| clang-format-version: '14' | |
| fallback-style: 'LLVM' | |
| check-path: 'cpp' | |
| - name: cpplint | |
| run: | | |
| pip install cpplint | |
| cpplint --recursive cpp | |
| - name: cmake-format | |
| run: | | |
| pip install cmakelang[YAML] | |
| find . -name "CMakeLists.txt" -not -path "*third_party/*" | xargs cmake-format -c .cmake-format.yaml --check | |
| build_test_job: | |
| if: github.repository == 'xorbitsai/xoscar' | |
| runs-on: ${{ matrix.os }} | |
| needs: lint | |
| env: | |
| CONDA_ENV: xoscar-test | |
| SELF_HOST_PYTHON: /root/miniconda3/envs/xoscar-test/bin/python | |
| SELF_HOST_CONDA: /root/miniconda3/condabin/conda | |
| defaults: | |
| run: | |
| shell: bash -l {0} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: ["ubuntu-latest", "macos-13", "windows-latest"] | |
| python-version: ["3.8", "3.9", "3.10", "3.11"] | |
| module: ["xoscar"] | |
| exclude: | |
| - { os: macos-13, python-version: 3.9} | |
| - { os: macos-13, python-version: 3.10} | |
| - { os: windows-latest, python-version: 3.9} | |
| - { os: windows-latest, python-version: 3.10} | |
| include: | |
| - { os: self-hosted, module: gpu, python-version: 3.11} | |
| steps: | |
| - name: Check out code | |
| uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| submodules: recursive | |
| - name: Add msbuild to PATH | |
| if: ${{ matrix.os == 'windows-latest'}} | |
| uses: microsoft/setup-msbuild@v1.1 | |
| - name: Set up conda ${{ matrix.python-version }} | |
| uses: conda-incubator/setup-miniconda@v3 | |
| if: ${{ matrix.module != 'gpu' }} | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| activate-environment: ${{ env.CONDA_ENV }} | |
| # Fix "version `GLIBCXX_3.4.30' not found (required by xoscar_store.cpython-311-x86_64-linux-gnu.so)" issue in Python 3.11 | |
| - name: Install libstdcxx-ng for Python 3.11 | |
| if: ${{ (matrix.module != 'gpu') && (matrix.os == 'ubuntu-latest') && (matrix.python-version == '3.11') }} | |
| run: | | |
| conda install -c conda-forge libstdcxx-ng | |
| - name: Install dependencies | |
| env: | |
| MODULE: ${{ matrix.module }} | |
| if: ${{ matrix.module != 'gpu' }} | |
| run: | | |
| pip install numpy scipy cython coverage flaky | |
| pip install -e ".[dev,extra]" | |
| working-directory: ./python | |
| - name: Install ucx dependencies | |
| if: ${{ (matrix.module != 'gpu') && (matrix.os == 'ubuntu-latest') && (matrix.python-version != '3.11') }} | |
| run: | | |
| conda install -c conda-forge -c rapidsai ucx-proc=*=cpu ucx ucx-py | |
| - name: Install fury | |
| if: ${{ (matrix.module != 'gpu') && (matrix.os == 'ubuntu-latest') && (matrix.python-version == '3.9') }} | |
| run: | | |
| pip install pyfury | |
| - name: Install on GPU | |
| if: ${{ matrix.module == 'gpu' }} | |
| run: | | |
| source activate ${{ env.CONDA_ENV }} | |
| conda install -y conda-forge::nccl=2.22.3 | |
| pip install --extra-index-url=https://pypi.nvidia.com cudf-cu12 | |
| pip install ucx-py-cu12 cloudpickle psutil tblib uvloop packaging "numpy<2.0.0" scipy cython coverage flaky | |
| python setup.py clean --all | |
| pip install -e ./ | |
| working-directory: ./python | |
| - name: Test with pytest | |
| env: | |
| MODULE: ${{ matrix.module }} | |
| run: | | |
| if [[ "$MODULE" == "xoscar" ]]; then | |
| pytest --timeout=1500 \ | |
| -W ignore::PendingDeprecationWarning \ | |
| --cov-config=setup.cfg --cov-report=xml --cov=xoscar xoscar --capture=no | |
| else | |
| source activate ${{ env.CONDA_ENV }} | |
| pytest -m cuda --cov-config=setup.cfg --cov-report=xml --cov=xoscar --capture=no | |
| fi | |
| working-directory: ./python | |
| - name: Report coverage data | |
| uses: codecov/codecov-action@v4 | |
| with: | |
| working-directory: ./python | |
| flags: unittests |