feat: Make RemoteRunner more resilient to CE issues #597
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Basic tests | |
| on: [push, pull_request] | |
| jobs: | |
| pre-commit: | |
| runs-on: ubuntu-latest | |
| if: github.event_name != 'push' || github.repository == 'DIRACGrid/DIRAC' | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| - name: Install pre-commit | |
| run: pip install pre-commit | |
| - name: Run pre-commit | |
| run: pre-commit run --all-files --show-diff-on-failure | |
| shellcheck: | |
| runs-on: ubuntu-latest | |
| if: github.event_name != 'push' || github.repository == 'DIRACGrid/DIRAC' | |
| timeout-minutes: 30 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Run shellcheck | |
| # TODO This should cover more than just tests/CI | |
| # Excluded codes related to sourcing files | |
| # SC1090: Can't follow non-constant source | |
| # SC1091: Not following sourced file | |
| run: | | |
| find tests/CI -name '*.sh' -print0 | xargs -0 -n1 shellcheck --exclude=SC1090,SC1091 --external-source | |
| pycodestyle: | |
| runs-on: ubuntu-latest | |
| if: github.event_name != 'push' || github.repository == 'DIRACGrid/DIRAC' | |
| timeout-minutes: 30 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| - name: Installing dependencies | |
| run: | | |
| python -m pip install pycodestyle | |
| - name: Run pycodestyle | |
| run: | | |
| if [[ "${REFERENCE_BRANCH}" != "" ]]; then | |
| git remote add upstream https://github.com/DIRACGrid/DIRAC.git | |
| git fetch --no-tags upstream "${REFERENCE_BRANCH}" | |
| git branch -vv | |
| git diff -U0 "upstream/${REFERENCE_BRANCH}" ':(exclude)tests/formatting/pep8_bad.py' | pycodestyle --diff | |
| fi | |
| check: | |
| runs-on: ubuntu-latest | |
| if: github.event_name != 'push' || github.repository == 'DIRACGrid/DIRAC' | |
| timeout-minutes: 30 | |
| defaults: | |
| # Activate the conda environment automatically in each step | |
| run: | |
| shell: bash -l {0} | |
| strategy: | |
| fail-fast: False | |
| matrix: | |
| command: | |
| # TODO These three tests fail on Python 3: | |
| # * `test_BaseType_Unicode` and `test_nestedStructure` fail due to | |
| # DISET's string and unicode types being poorly defined | |
| - pytest --runslow -k 'not test_BaseType_Unicode and not test_nestedStructure' | |
| - pylint -j 0 -E src/ | |
| - mypy | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Fail-fast for outdated pipelines | |
| run: .github/workflows/fail-fast.sh | |
| - uses: conda-incubator/setup-miniconda@master | |
| with: | |
| environment-file: environment.yml | |
| miniforge-variant: Mambaforge | |
| use-mamba: true | |
| - name: Run tests | |
| run: | | |
| # FIXME: The unit tests currently only work with editable installs | |
| pip install -e .[server,testing] | |
| ${{ matrix.command }} | |
| env: | |
| REFERENCE_BRANCH: ${{ github['base_ref'] || github['head_ref'] }} | |
| check-py39: | |
| runs-on: ubuntu-latest | |
| if: github.event_name != 'push' || github.repository == 'DIRACGrid/DIRAC' | |
| timeout-minutes: 30 | |
| defaults: | |
| # Activate the conda environment automatically in each step | |
| run: | |
| shell: bash -l {0} | |
| strategy: | |
| fail-fast: False | |
| matrix: | |
| command: | |
| # TODO These three tests fail on Python 3: | |
| # * `test_BaseType_Unicode` and `test_nestedStructure` fail due to | |
| # DISET's string and unicode types being poorly defined | |
| - pytest --runslow -k 'not test_BaseType_Unicode and not test_nestedStructure' | |
| - pylint -j 0 -E src/ | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Fail-fast for outdated pipelines | |
| run: .github/workflows/fail-fast.sh | |
| - name: fix python3.9 | |
| run: | | |
| sed -i "s/python =3.11/python =3.9/g" environment.yml | |
| - uses: conda-incubator/setup-miniconda@master | |
| with: | |
| environment-file: environment.yml | |
| miniforge-variant: Mambaforge | |
| use-mamba: true | |
| - name: Run tests | |
| run: | | |
| # FIXME: The unit tests currently only work with editable installs | |
| pip install -e .[server,testing] | |
| ${{ matrix.command }} | |
| env: | |
| REFERENCE_BRANCH: ${{ github['base_ref'] || github['head_ref'] }} | |
| pylint-py27: | |
| name: Pylint for Python 2.7 in Pilot files | |
| runs-on: ubuntu-latest | |
| if: github.event_name != 'push' || github.repository == 'DIRACGrid/DIRAC' | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Fail-fast for outdated pipelines | |
| run: .github/workflows/fail-fast.sh | |
| - name: prepare environment | |
| run: | | |
| conda config --set add_pip_as_python_dependency false | |
| conda create -c conda-forge -n test-env python=2.7 pylint=1.9.2 astroid=1.6.5 | |
| - name: run pilot wrapper test | |
| run: | | |
| eval "$(conda shell.bash hook)" && conda activate test-env | |
| pylint -j 0 -E \ | |
| tests/Integration/WorkloadManagementSystem/Test_GenerateAndExecutePilotWrapper.py \ | |
| src/DIRAC/WorkloadManagementSystem/Utilities/PilotWrapper.py \ | |
| src/DIRAC/Resources/Computing/BatchSystems/*.py |