Skip to content

Introduce Rust for replacing parts of Python code #2338

Introduce Rust for replacing parts of Python code

Introduce Rust for replacing parts of Python code #2338

name: Build and test
on:
push:
branches: [ "master" ]
tags: "v*.*.*"
pull_request:
branches: [ "master" ]
workflow_dispatch:
env:
BUILD_TYPE: Release
PIP_BREAK_SYSTEM_PACKAGES: 1
permissions:
contents: write
checks: write
jobs:
##############################################################
## Release creation for tag pushes
##############################################################
maybe_create_release:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
if: startsWith(github.ref, 'refs/tags/')
- name: Create Release
if: startsWith(github.ref, 'refs/tags/')
uses: softprops/action-gh-release@v2
id: create_release
with:
draft: true
generate_release_notes: true
append_body: true
body: |
This is a development release. Consider alpha / unstable until v1.0.0.
## Installation
Choose between Python wheels, portable folder archives and packages/installers per platform.
For details, refer to the [full installation instructions](https://github.com/SanderVocke/shoopdaloop/blob/master/INSTALL.md).
- name: Store asset upload URL
if: startsWith(github.ref, 'refs/tags/')
shell: bash
run: echo "${{ steps.create_release.outputs.upload_url }}" > release_upload_assets_url.txt
- name: Upload asset upload URL
if: startsWith(github.ref, 'refs/tags/')
uses: actions/upload-artifact@v4
with:
name: release_upload_assets_url
path: release_upload_assets_url.txt
##############################################################
## Linux
##############################################################
build_linux:
strategy:
fail-fast: false
matrix:
kind:
- name: linux_release
build_type: 'Release'
pyinstaller: true
package_kind: linuxes
upload_release_assets: true
- name: linux_debug
build_type: 'Debug'
# - name: linux_asan
# # Note: ASAN build does not have a test because it is not trivial to deploy
# # an asan-built program on another system. The build is here to ensure the
# # build process still works. To use ASAN, compile like this natively.
# build_type: 'Debug'
# cmake_opts: '"ENABLE_ASAN" = "On"'
# - name: linux_coverage
# build_type: 'Debug'
# cmake_opts: '"ENABLE_COVERAGE" = "On"'
# coverage: true
container: [ "docker.io/sandervocke/shoopdaloop_build_base_debian_bullseye_x86_64:latest" ]
container_options: [ "--user root --workdir /" ]
arch: [ x86_64 ]
manylinux_arch: [ x86_64 ]
runs-on: ubuntu-latest
container:
image: ${{ matrix.container }}
options: ${{ matrix.container_options }}
needs: maybe_create_release
#if: false
steps:
- name: Checkout
uses: actions/checkout@v4
with:
submodules: recursive
- name: Setup Shell Wrapper
uses: sandervocke/setup-shell-wrapper@v1
- name: Use regular shell
shell: bash
run: echo "WRAP_SHELL=bash --noprofile --norc -eo pipefail" >> $GITHUB_ENV
- name: Build
uses: ./.github/actions/build_toplevel
with:
name: ${{ matrix.kind.name }}
cmake_build_type: ${{ matrix.kind.build_type }}
cmake_opts: ${{ matrix.kind.cmake_opts }}
prepare_kind: 'linux'
python: python3.9
rename_wheel_sed: "'s/([^\\.]+)[\\.]+whl/\\1.whl/g'" # remove double dot
pyinstaller: ${{ matrix.kind.pyinstaller }}
package_kind: ${{ matrix.kind.package_kind }}
coverage: ${{ matrix.kind.coverage }}
upload_release_assets: ${{ (matrix.kind.upload_release_assets && startsWith(github.ref, 'refs/tags/')) && 'true' || 'false' }}
# - name: Setup tmate session
# if: failure()
# uses: SanderVocke/action-tmate@master
test_linux:
strategy:
fail-fast: false
matrix:
kind:
- name: release_wheel_debian_stable
python: python3
python_pip_args: -m pip
python_pytest_args: -m pytest
container: docker.io/sandervocke/shoopdaloop_run_base_debian_latest_x86_64:latest
package: wheel_linux_release
- name: debug_wheel_debian_stable
python: python3
python_pip_args: -m pip
python_pytest_args: -m pytest
container: docker.io/sandervocke/shoopdaloop_run_base_debian_latest_x86_64:latest
package: wheel_linux_debug
# - name: release_deb_ubuntu_latest
# python: python3
# python_pip_args: -m pip
# python_pytest_args: -m pytest
# container: docker.io/sandervocke/shoopdaloop_run_base_debian_latest_x86_64:latest
# package: package_linux_release_deb
# - name: release_pyinst_ubuntu_latest
# python: python3
# python_pip_args: -m pip
# python_pytest_args: -m pytest
# container: docker.io/sandervocke/shoopdaloop_run_base_debian_latest_x86_64:latest
# package: pyinstaller_linux_release
# - name: release_pacman_arch
# python: python
# python_pip_args: -m pip
# python_pytest_args: -m pytest
# container: docker.io/sandervocke/shoopdaloop_run_base_arch_x86_64:latest
# package: package_linux_release_pacman
# - name: release_rpm_fedora
# python: python3
# python_pip_args: -m pip
# python_pytest_args: -m pytest
# container: docker.io/sandervocke/shoopdaloop_run_base_fedora_x86_64:latest
# package: package_linux_release_rpm
# - name: coverage_debian_bullseye
# python: python3
# python_shoopdaloop_args: -m coverage run --source=shoopdaloop
# python_pip_args: -m pip
# python_pytest_args: -m coverage run --source=shoopdaloop -m pytest
# container: docker.io/sandervocke/shoopdaloop_build_base_debian_bullseye_x86_64:latest
# package: wheel_linux_coverage
# coverage: true
# run_cmd_prefix: >-
# BASEDIR=$(pwd)
# EXCLUDE='/*/third_party/* /*/catch2/*'
# REPORTDIR=coverage_reports
# BUILDDIR=$(realpath build/cp*/)
# ORI_BUILD_DIR=$(cat ori_build_dir.txt)
# DO_GENHTML=0
# SHOOP_LOG=QoverageCollectorFactory=debug
# QML_IMPORT_PATH=$(~/.local/bin/qoverage --importpath)
# scripts/run_and_generate_coverage.sh
runs-on: ubuntu-latest
needs: build_linux
#if: false
container:
image: ${{ matrix.kind.container }}
options: --user root --workdir / # Note: this gets disregarded if container is null
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Setup Shell Wrapper
uses: sandervocke/setup-shell-wrapper@v1
- name: Use regular shell
shell: bash
run: echo "WRAP_SHELL=bash --noprofile --norc -eo pipefail" >> $GITHUB_ENV
- name: Test
uses: ./.github/actions/test_toplevel
with:
name: ${{ matrix.kind.name }}
package: ${{ matrix.kind.package }}
asan: ${{ matrix.kind.asan }}
coverage: ${{ matrix.kind.coverage }}
python_pip_args: ${{ matrix.kind.python_pip_args }}
python: ${{ matrix.kind.python }}
run_cmd_prefix: ${{ matrix.kind.run_cmd_prefix }}
python_pytest_args: ${{ matrix.kind.python_pytest_args }}
qpa_platform: vnc
pyinstaller: ${{ matrix.kind.pyinstaller }}
codecov_token: ${{ secrets.CODECOV_TOKEN }}
##############################################################
## Linux (ARM emulated)
##############################################################
# build_linux_arm:
# strategy:
# fail-fast: false
# matrix:
# container: [ "docker.io/sandervocke/shoopdaloop_build_base_debian_bullseye_arm64:latest" ]
# arch: [ arm64 ]
# manylinux_arch: [ aarch64 ]
# kind:
# - name: linux_arm_release
# build_type: 'Release'
# pyinstaller: true
# package_kind: linuxes
# upload_release_assets: true
# - name: linux_arm_debug
# build_type: 'Debug'
# runs-on: ubuntu-latest
# needs: maybe_create_release
# #if: false
# steps:
# - name: Checkout
# uses: actions/checkout@v4
# with:
# submodules: recursive
# - name: Start QEMU container
# if: ${{ matrix.container }}
# uses: sandervocke/setup-qemu-container@v1
# with:
# container: ${{ matrix.container }}
# arch: ${{ matrix.arch }}
# - name: Setup Shell Wrapper
# uses: sandervocke/setup-shell-wrapper@v1
# - name: Use container shell
# shell: bash
# run: echo "WRAP_SHELL=run-in-container.sh --shell bash" >> $GITHUB_ENV
# - name: Build
# uses: ./.github/actions/build_toplevel
# with:
# name: ${{ matrix.kind.name }}
# cmake_build_type: ${{ matrix.kind.build_type }}
# cmake_opts: ${{ matrix.kind.cmake_opts }}
# prepare_kind: 'linux'
# python: python3.9
# rename_wheel_sed: "'s/([^\\.]+)[\\.]+whl/\\1.whl/g'" # remove double dot
# pyinstaller: ${{ matrix.kind.pyinstaller }}
# package_kind: ${{ matrix.kind.package_kind }}
# coverage: ${{ matrix.kind.coverage }}
# upload_release_assets: ${{ (matrix.kind.upload_release_assets && startsWith(github.ref, 'refs/tags/')) && 'true' || 'false' }}
# test_linux_arm:
# strategy:
# fail-fast: false
# matrix:
# container_arch: [ arm64 ]
# kind:
# - name: release_wheel_debian_bullseye_arm
# python: python3
# python_pip_args: -m pip
# python_pytest_args: -m pytest
# container: docker.io/sandervocke/shoopdaloop_run_base_debian_latest_arm64:latest
# package: wheel_linux_arm_release
# - name: release_deb_ubuntu_latest_arm
# python: python3
# python_pip_args: -m pip
# python_pytest_args: -m pytest
# container: docker.io/sandervocke/shoopdaloop_run_base_ubuntu_latest_arm64:latest
# package: package_linux_arm_release_deb
# install_with_sudo: 'true'
# runs-on: ubuntu-latest
# needs: build_linux_arm
# #if: false
# steps:
# - name: Checkout
# uses: actions/checkout@v4
# - name: Start QEMU container
# uses: sandervocke/setup-qemu-container@v1
# with:
# container: ${{ matrix.kind.container }}
# arch: ${{ matrix.container_arch }}
# - name: Setup Shell Wrapper
# uses: sandervocke/setup-shell-wrapper@v1
# - name: Use container shell
# shell: bash
# run: echo "WRAP_SHELL=run-in-container.sh --shell bash" >> $GITHUB_ENV
# - name: Test
# uses: ./.github/actions/test_toplevel
# with:
# name: ${{ matrix.kind.name }}
# package: ${{ matrix.kind.package }}
# asan: ${{ matrix.kind.asan }}
# coverage: ${{ matrix.kind.coverage }}
# python_pip_args: ${{ matrix.kind.python_pip_args }}
# python: ${{ matrix.kind.python }}
# run_cmd_prefix: ${{ matrix.kind.run_cmd_prefix }}
# python_pytest_args: ${{ matrix.kind.python_pytest_args }}
# qpa_platform: vnc
# pyinstaller: ${{ matrix.kind.pyinstaller }}
# codecov_token: ${{ secrets.CODECOV_TOKEN }}
# install_with_sudo: ${{ matrix.kind.install_with_sudo }}
# do_qml_tests: false # Too slow on ARM emulation
# # ##############################################################
# # ## MacOS
# # ##############################################################
build_macos:
strategy:
fail-fast: false
matrix:
kind:
# - name: macos_release
# build_type: 'Release'
# appbundle: true
# package_kind: dmg
# os: macos-12
# upload_release_assets: true
# - name: macos_debug
# build_type: 'Debug'
# os: macos-12
# - name: macos_release_arm
# build_type: 'Release'
# appbundle: true
# package_kind: dmg
# os: macos-14
# upload_release_assets: true
- name: macos_debug_arm
build_type: 'Debug'
os: macos-14
runs-on: ${{ matrix.kind.os }}
needs: maybe_create_release
#if: false
steps:
- name: Checkout
uses: actions/checkout@v4
with:
submodules: recursive
- name: Setup Shell Wrapper
uses: sandervocke/setup-shell-wrapper@v1
- name: Use regular shell
shell: bash
run: echo "WRAP_SHELL=bash --noprofile --norc -eo pipefail" >> $GITHUB_ENV
- name: Build
uses: ./.github/actions/build_toplevel
with:
name: ${{ matrix.kind.name }}
cmake_build_type: ${{ matrix.kind.build_type }}
cmake_opts: ${{ matrix.kind.cmake_opts }}
prepare_kind: 'macos'
python: python3.9
pyinstaller: ${{ matrix.kind.pyinstaller }}
appbundle: ${{ matrix.kind.appbundle }}
package_kind: ${{ matrix.kind.package_kind }}
coverage: ${{ matrix.kind.coverage }}
# Note: for some reason, an incorrect MacOS identifier is generated (macos_11 instead of macos_11_0).
# Also remove the double dot if any.
rename_wheel_sed: "'s/(.*-macosx_[1-9]+_)(x86_64|arm64|universal2)[\\.]*whl/\\10_\\2.whl/g'"
upload_release_assets: ${{ (matrix.kind.upload_release_assets && startsWith(github.ref, 'refs/tags/')) && 'true' || 'false' }}
# - name: Setup tmate session
# if: failure()
# uses: SanderVocke/action-tmate@master
test_macos:
strategy:
fail-fast: false
matrix:
kind:
# - name: release_whl_macos12
# python: python3
# python_pip_args: -m pip
# python_pytest_args: -m pytest
# container: null
# os: macos-12
# package: wheel_macos_release
# qpa_platform: offscreen
# - name: release_whl_macos13
# python: python3
# python_pip_args: -m pip
# python_pytest_args: -m pytest
# container: null
# os: macos-13
# package: wheel_macos_release
# qpa_platform: offscreen
# - name: release_whl_macos14
# python: python3
# python_pip_args: -m pip
# python_pytest_args: -m pytest
# container: null
# os: macos-14
# package: wheel_macos_release_arm
# qpa_platform: offscreen
- name: debug_whl_macos14
python: python3
python_pip_args: -m pip
python_pytest_args: -m pytest
container: null
os: macos-14
package: wheel_macos_debug_arm
qpa_platform: offscreen
# - name: release_dmg_macos14
# python: python3
# python_pip_args: -m pip
# python_pytest_args: -m pytest
# container: null
# os: macos-14
# package: package_macos_release_arm
# qpa_platform: offscreen
runs-on: ${{ matrix.kind.os }}
needs: build_macos
#if: false
container:
image: ${{ matrix.kind.container }}
options: --user root --workdir / # Note: this gets disregarded if container is null
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Setup Shell Wrapper
uses: sandervocke/setup-shell-wrapper@v1
- name: Use regular shell
shell: bash
run: echo "WRAP_SHELL=bash --noprofile --norc -eo pipefail" >> $GITHUB_ENV
- name: Test
uses: ./.github/actions/test_toplevel
with:
name: ${{ matrix.kind.name }}
package: ${{ matrix.kind.package }}
asan: ${{ matrix.kind.asan }}
coverage: ${{ matrix.kind.coverage }}
python_pip_args: ${{ matrix.kind.python_pip_args }}
python: ${{ matrix.kind.python }}
run_cmd_prefix: ${{ matrix.kind.run_cmd_prefix }}
python_pytest_args: ${{ matrix.kind.python_pytest_args }}
qpa_platform: ${{ matrix.kind.qpa_platform }}
pyinstaller: ${{ matrix.kind.pyinstaller }}
codecov_token: ${{ secrets.CODECOV_TOKEN }}
# # ##############################################################
# # ## Windows
# # ##############################################################
build_windows:
strategy:
fail-fast: false
matrix:
kind:
# - name: windows_release
# os: windows-2022
# python: python.exe
# pip: python.exe -m pip
# container_image: null
# prepare: 'windows'
# build_type: 'Release'
# pyinstaller: true
# package_kind: innosetup
# rename_wheel_sed: "'s/([^\\.]+)[\\.]+whl/\\1.whl/g'" # remove double dot
# upload_release_assets: true
- name: windows_debug
os: windows-2022
python: python.exe
pip: python.exe -m pip
container_image: null
prepare: 'windows'
build_type: 'Debug'
rename_wheel_sed: "'s/([^\\.]+)[\\.]+whl/\\1.whl/g'" # remove double dot
# cmake_opts: '"BACKEND_JACK" = "Off", "HAVE_LV2" = "Off"'
runs-on: ${{ matrix.kind.os}}
needs: maybe_create_release
container:
image: ${{ matrix.kind.container_image }}
options: --user root --workdir /
steps:
- name: Checkout
uses: actions/checkout@v4
with:
submodules: recursive
- name: Setup Shell Wrapper
uses: sandervocke/setup-shell-wrapper@v1
- name: Use regular shell
shell: bash
run: echo "WRAP_SHELL=bash --noprofile --norc -eo pipefail" >> $GITHUB_ENV
- name: Build
uses: ./.github/actions/build_toplevel
with:
name: ${{ matrix.kind.name }}
cmake_build_type: ${{ matrix.kind.build_type }}
cmake_opts: ${{ matrix.kind.cmake_opts }}
prepare_kind: ${{ matrix.kind.prepare }}
python: ${{ matrix.kind.python }}
rename_wheel_sed: ${{ matrix.kind.rename_wheel_sed }}
pyinstaller: ${{ matrix.kind.pyinstaller }}
package_kind: ${{ matrix.kind.package_kind }}
coverage: ${{ matrix.kind.coverage }}
upload_release_assets: ${{ (matrix.kind.upload_release_assets && startsWith(github.ref, 'refs/tags/')) && 'true' || 'false' }}
# - name: Setup tmate session
# if: failure()
# uses: SanderVocke/action-tmate@master
test_windows:
strategy:
fail-fast: false
matrix:
kind:
# - name: release_windows
# python: python.exe
# python_pip_args: -m pip
# python_pytest_args: -m pytest
# container: null
# os: windows-2022
# package: wheel_windows_release
# qpa_platform: offscreen
# pathconvert: "| cygpath -u -f -"
# - name: release_windows_innosetup
# python: python.exe
# python_pip_args: -m pip
# python_pytest_args: -m pytest
# container: null
# os: windows-2022
# package: package_windows_release
# qpa_platform: offscreen
# pathconvert: "| cygpath -u -f -"
- name: debug_windows
python: python.exe
python_pip_args: -m pip
python_pytest_args: -m pytest
container: null
os: windows-2022
package: wheel_windows_debug
qpa_platform: offscreen
pathconvert: "| cygpath -u -f -"
runs-on: ${{ matrix.kind.os }}
needs: build_windows
#if: false
container:
image: ${{ matrix.kind.container }}
options: --user root --workdir / # Note: this gets disregarded if container is null
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Setup Shell Wrapper
uses: sandervocke/setup-shell-wrapper@v1
- name: Use regular shell
shell: bash
run: echo "WRAP_SHELL=bash --noprofile --norc -eo pipefail" >> $GITHUB_ENV
- name: Test
uses: ./.github/actions/test_toplevel
with:
name: ${{ matrix.kind.name }}
package: ${{ matrix.kind.package }}
asan: ${{ matrix.kind.asan }}
coverage: ${{ matrix.kind.coverage }}
python_pip_args: ${{ matrix.kind.python_pip_args }}
python: ${{ matrix.kind.python }}
run_cmd_prefix: ${{ matrix.kind.run_cmd_prefix }}
python_pytest_args: ${{ matrix.kind.python_pytest_args }}
qpa_platform: ${{ matrix.kind.qpa_platform }}
pyinstaller: ${{ matrix.kind.pyinstaller }}
codecov_token: ${{ secrets.CODECOV_TOKEN }}
# ##############################################################
# ## Crash collection
# ##############################################################
collect_crashes:
runs-on: ubuntu-latest
needs:
- build_linux
- test_linux
# - build_linux_arm
# - test_linux_arm
# - build_macos
# - test_macos
# - build_windows
# - test_windows
if: ${{ always() }}
container:
image: archlinux:latest
options: --user root --workdir /
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Setup Shell Wrapper
uses: sandervocke/setup-shell-wrapper@v1
- name: Use regular shell
shell: bash
run: echo "WRAP_SHELL=bash --noprofile --norc -eo pipefail" >> $GITHUB_ENV
- name: Download artifacts
uses: actions/download-artifact@v4
with:
pattern: crashdumps_*_${{ github.run_number }}
path: crashdumps
- name: Count crash dumps
id: count
shell: wrap-shell {0}
run: |
mkdir -p crashdumps
echo "crashdumps=$(ls crashdumps/crashdumps_* | wc -l)" | tee -a $GITHUB_OUTPUT
pacman --noconfirm -Syy
pacman --noconfirm -S --needed tree git
- name: Download debug information
uses: actions/download-artifact@v4
if: ${{ steps.count.outputs.crashdumps != 0 }}
with:
path: debuginfo
pattern: debuginfo_*
merge-multiple: true
- name: Cache minidump-stackwalk
id: cache-minidump
uses: actions/cache@v4
with:
path: minidump-stackwalk
key: minidump-stackwalk-1
- name: Get minidump package sources
if: ${{ steps.count.outputs.crashdumps != 0 && steps.cache-minidump.outputs.cache-hit != 'true' }}
shell: wrap-shell {0}
run: |
pacman --noconfirm -Syy
pacman --noconfirm -S --needed git base-devel && git clone https://aur.archlinux.org/minidump-stackwalk.git
- name: Makepkg Build
id: makepkg
if: ${{ steps.count.outputs.crashdumps != 0 && steps.cache-minidump.outputs.cache-hit != 'true' }}
uses: edlanglois/pkgbuild-action@v1
with:
pkgdir: minidump-stackwalk
- name: Install minidump package
if: ${{ steps.count.outputs.crashdumps != 0 }}
shell: wrap-shell {0}
run: |
pacman --noconfirm -U minidump-stackwalk/minidump-stackwalk*.pkg*
- name: Process dumps
if: ${{ steps.count.outputs.crashdumps != 0 }}
shell: wrap-shell {0}
run: |
tree crashdumps
if [ -e debuginfo ]; then
cp -r debuginfo crashdumps/debuginfo
echo "found_debuginfo=true" | tee -a $GITHUB_ENV
fi
mkdir -p debuginfo/breakpad_symbols
mkdir -p crashdumps/debuginfo
for dump in $(find crashdumps -name "*.dmp"); do
echo ""
echo "================================================="
echo "Processing $dump..."
echo "================================================="
minidump-stackwalk $dump debuginfo/breakpad_symbols || true | tee $dump.stackwalk
done
- name: Upload crash dumps to Bugsplat
if: ${{ (success() || failure()) && steps.count.outputs.crashdumps != 0 }}
shell: wrap-shell {0}
run: |
for dump in $(find crashdumps -name "*.dmp"); do
echo ""
echo "================================================="
echo "Uploading $dump..."
echo "================================================="
bash ./scripts/upload_crash_info.sh $dump git-${{ github.sha }} "sandervocke@gmail.com" "Found on GitHub Actions run: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" ${dump}.moreinfo
done
- uses: actions/setup-node@v4
if: ${{ (success() || failure()) && steps.count.outputs.crashdumps != 0 }}
with:
node-version: 19
- name: Upload BugSplat debug symbols
if: ${{ (success() || failure()) && steps.count.outputs.crashdumps != 0 && env.found_debuginfo == 'true' }}
uses: SanderVocke/symbol-upload@test_action
with:
clientId: "${{ secrets.BUGSPLAT_CLIENT_ID }}"
clientSecret: "${{ secrets.BUGSPLAT_CLIENT_SECRET }}"
database: shoopdaloop
application: shoopdaloop
version: git-${{ github.sha }}
files: "**/*.sym"
directory: crashdumps/debuginfo
- name: Upload crash dumps as artifact
if: ${{ (success() || failure()) && steps.count.outputs.crashdumps != 0 }}
uses: actions/upload-artifact@v4
with:
name: crashdumps_summary
path: crashdumps