[gym_jiminy/common] Speed-up env pipeline. #3324
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Manylinux CI (PyPi deploy) | |
| on: | |
| # Trigger the workflow on push on the master branch, or for any pull request | |
| push: | |
| branches: | |
| - master | |
| pull_request: | |
| jobs: | |
| build-manylinux: | |
| name: (${{ matrix.PYTHON_VERSION }}) Build wheels. | |
| strategy: | |
| matrix: | |
| PYTHON_VERSION: ['cp38', 'cp39', 'cp310', 'cp311'] | |
| runs-on: ubuntu-20.04 | |
| container: 'quay.io/pypa/manylinux_2_28_x86_64:latest' | |
| defaults: | |
| run: | |
| # Using bash enables automatic sourcing `.bashrc` and fail-fast behavior | |
| shell: bash -ieo pipefail {0} | |
| env: | |
| BUILD_TYPE: "Release" | |
| CMAKE_CXX_FLAGS: "-DEIGEN_MPL2_ONLY" | |
| ##################################################################################### | |
| steps: | |
| - name: Checkout jiminy | |
| uses: actions/checkout@v3 | |
| ##################################################################################### | |
| - name: Setup minimal build environment | |
| run: | | |
| pythonLocation=$(find /opt/python -maxdepth 1 -name \ | |
| "$(echo "${{ matrix.PYTHON_VERSION }}*" | sed -e 's/\.//g')" -print -quit) | |
| echo "export PATH=\"${pythonLocation}/bin:\$PATH\"" >> $HOME/.bashrc | |
| source $HOME/.bashrc | |
| git config --global advice.detachedHead false | |
| PYTHON_EXECUTABLE="${pythonLocation}/bin/python" | |
| echo "PYTHON_EXECUTABLE=${PYTHON_EXECUTABLE}" >> $GITHUB_ENV | |
| echo "RootDir=${GITHUB_WORKSPACE}" >> $GITHUB_ENV | |
| echo "InstallDir=${GITHUB_WORKSPACE}/install" >> $GITHUB_ENV | |
| "${PYTHON_EXECUTABLE}" -m pip install setuptools wheel "pip>=20.3" | |
| "${PYTHON_EXECUTABLE}" -m pip install "numpy>=1.21" "numba>=0.54.0" | |
| "${PYTHON_EXECUTABLE}" -m pip install twine cmake | |
| - name: Build project dependencies | |
| run: | | |
| ./build_tools/build_install_deps_unix.sh | |
| ##################################################################################### | |
| - name: Build and install Jiminy | |
| run: | | |
| unset Boost_ROOT | |
| # Build jiminy | |
| mkdir "$RootDir/build" | |
| cd "$RootDir/build" | |
| export LD_LIBRARY_PATH="$InstallDir/lib:$InstallDir/lib64:/usr/local/lib" | |
| cmake "$RootDir" -DCMAKE_INSTALL_PREFIX="$InstallDir" -DCMAKE_PREFIX_PATH="$InstallDir" \ | |
| -DCMAKE_INTERPROCEDURAL_OPTIMIZATION=ON -DCMAKE_VERBOSE_MAKEFILE=ON \ | |
| -DBOOST_ROOT="$InstallDir" -DBoost_INCLUDE_DIR="$InstallDir/include" \ | |
| -DBoost_NO_SYSTEM_PATHS=TRUE -DBoost_NO_BOOST_CMAKE=TRUE \ | |
| -DBoost_USE_STATIC_LIBS=ON -DPYTHON_EXECUTABLE="${PYTHON_EXECUTABLE}" \ | |
| -DBUILD_TESTING=ON -DBUILD_EXAMPLES=ON \ | |
| -DBUILD_PYTHON_INTERFACE=ON -DINSTALL_GYM_JIMINY=ON \ | |
| -DCMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS}" -DCMAKE_BUILD_TYPE="$BUILD_TYPE" | |
| make -j2 | |
| # Bundle the boost python dependencies with jiminy | |
| mkdir -p "$RootDir/build/pypi/jiminy_py/src/jiminy_py" | |
| PYTHON_VERSION="$(${PYTHON_EXECUTABLE} -c \ | |
| "import sysconfig; print(sysconfig.get_config_var('py_version_short'))")" | |
| \cp -r -H "$InstallDir/lib/python${PYTHON_VERSION}/site-packages/." \ | |
| "$RootDir/build/pypi/jiminy_py/src/jiminy_py/core" | |
| # Install jiminy along with its dependencies | |
| make install | |
| # Strip all symbol table and relocation information from python bindings and shared libraries | |
| find "$RootDir/build/pypi/" -name "*.so*" -print -exec strip -s {} + | |
| find "$InstallDir/lib"* -name "*.so*" -maxdepth 1 -print -exec strip -s {} + | |
| ##################################################################################### | |
| - name: Generate Python Pip wheels | |
| run: | | |
| export LD_LIBRARY_PATH="$InstallDir/lib:$InstallDir/lib64:/usr/local/lib" | |
| # Generate stubs | |
| stubgen -p jiminy_py -o $RootDir/build/pypi/jiminy_py/src | |
| "${PYTHON_EXECUTABLE}" "$RootDir/build_tools/stub_gen.py" \ | |
| -o $RootDir/build/stubs --ignore-invalid=all jiminy_py | |
| \cp $RootDir/build/stubs/jiminy_py-stubs/core/__init__.pyi \ | |
| $RootDir/build/pypi/jiminy_py/src/jiminy_py/core/core.pyi | |
| # Generate wheels | |
| cd "$RootDir/build" | |
| cmake . -DCOMPONENT=pypi -P ./cmake_install.cmake | |
| # Bundle non-standard dependencies with the wheels for optimal portability. | |
| # Note that `--strip` option is not used but rather done manually because the | |
| # built-in option corrupts the shared library for some reason... | |
| "${PYTHON_EXECUTABLE}" "$RootDir/build_tools/wheel_repair_linux.py" repair \ | |
| -w "$RootDir/build/wheelhouse" "$RootDir/build/pypi/dist/jiminy_py/"*.whl | |
| - name: Upload the wheel of Jiminy for Linux | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: wheelhouse | |
| path: build/wheelhouse | |
| - name: Upload the wheels of Gym Jiminy | |
| if: success() && matrix.PYTHON_VERSION == 'cp38' | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: wheelhouse | |
| path: build/pypi/dist/gym_jiminy | |
| ##################################################################################### | |
| test-user-linux: | |
| name: (${{ matrix.os }}) Build a pip extension and run the unit tests. | |
| needs: build-manylinux | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| os: [ubuntu-22.04] | |
| defaults: | |
| run: | |
| shell: bash -ieo pipefail {0} | |
| env: | |
| BUILD_TYPE: "Release" | |
| ##################################################################################### | |
| steps: | |
| - name: Checkout jiminy | |
| uses: actions/checkout@v3 | |
| - name: Setup minimal build environment | |
| run: | | |
| PYTHON_EXECUTABLE="/usr/bin/python3" | |
| echo "PYTHON_EXECUTABLE=${PYTHON_EXECUTABLE}" >> $GITHUB_ENV | |
| echo "RootDir=${GITHUB_WORKSPACE}" >> $GITHUB_ENV | |
| echo "InstallDir=${GITHUB_WORKSPACE}/install" >> $GITHUB_ENV | |
| echo "/home/runner/.local/bin" >> $GITHUB_PATH | |
| - name: Installing requirements | |
| run: | | |
| sudo env "PATH=$PATH" "$RootDir/build_tools/easy_install_deps_ubuntu.sh" | |
| "${PYTHON_EXECUTABLE}" -m pip install --prefer-binary pillow scipy | |
| ##################################################################################### | |
| - name: Download the wheels previously generated | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: wheelhouse | |
| path: wheelhouse | |
| - name: Install the best wheel among the previously generated | |
| run: | | |
| # extract the "best" wheel for our system out of the wheelhouse, using pip's own resolution | |
| # we cannot directly install with find-links because it does not guarantee precedence of local wheels | |
| # see https://github.com/pypa/pip/issues/2589 (pip>=21.1) | |
| "${PYTHON_EXECUTABLE}" -m pip download \ | |
| --no-index --find-links="$RootDir/wheelhouse" --no-deps --dest="$RootDir/dist" jiminy_py | |
| # rely on bash to expand the full wheel name | |
| "${PYTHON_EXECUTABLE}" -m pip install --prefer-binary $RootDir/dist/jiminy_py* | |
| ##################################################################################### | |
| - name: Build extension module | |
| run: | | |
| mkdir -p "$RootDir/core/examples/external_project/build" | |
| cd "$RootDir/core/examples/external_project/build" | |
| cmake "$RootDir/core/examples/external_project/" -DCMAKE_INSTALL_PREFIX="$InstallDir" \ | |
| -DCMAKE_PREFIX_PATH="$InstallDir" -DCMAKE_VERBOSE_MAKEFILE=ON \ | |
| -DPYTHON_EXECUTABLE="${PYTHON_EXECUTABLE}" -DCMAKE_BUILD_TYPE="${BUILD_TYPE}" | |
| make install | |
| "$InstallDir/bin/pip_double_pendulum" | |
| - name: Run unit tests on user environment | |
| run: | | |
| cd "${GITHUB_WORKSPACE}/python/jiminy_py/unit_py" | |
| "${PYTHON_EXECUTABLE}" -m unittest discover -v | |
| ##################################################################################### | |
| publish-pypi-manylinux: | |
| name: Publish on PyPi the wheel for Linux of jiminy_py and gym_jiminy | |
| needs: test-user-linux | |
| runs-on: ubuntu-20.04 | |
| if: github.repository == 'duburcqa/jiminy' && github.event_name == 'push' && github.ref == 'refs/heads/master' | |
| steps: | |
| - name: Download the wheels previously generated | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: wheelhouse | |
| path: wheelhouse | |
| - name: Publish the wheels on PyPi | |
| uses: pypa/gh-action-pypi-publish@v1.7.1 | |
| with: | |
| user: __token__ | |
| password: ${{ secrets.PYPI_DEPLOY }} | |
| packages-dir: wheelhouse | |
| verify-metadata: false |