build-test-publish.yml

# Builds on all branches & PRs and packages for MacOS and Windows.
# Deploys to PyPi for tags.
name: Build, test and publish

on:
  push:
    branches:
      - master
      - l10n_master
      - release
      - version/*
  pull_request:
    branches-ignore:
      - l10n_master
  create:
    tags:
      - '*'

jobs:
  typechecks:
    runs-on: ubuntu-latest
    name: Type checks
    strategy:
      max-parallel: 5
      matrix:
        python-version: [ "3.9", "3.10", "3.11", "3.12" ]
    steps:
    - uses: actions/checkout@v4
    - uses: theCapypara/mypy-check@rust-support
      name: Run type checks
      with:
        mypy_flags: '--config-file mypy.ini'
        requirements: '-r requirements_type_checks.txt'
        python_version: '${{ matrix.python-version }}'

  linting:
    runs-on: ubuntu-latest
    name: Linting
    steps:
    - uses: actions/checkout@v4
    - name: Install gobject-introspection
      run: sudo apt-get install -y gobject-introspection libgirepository1.0-dev
    - name: Set up Python 3.11
      uses: actions/setup-python@v5
      with:
        python-version: "3.11"
    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        pip install -r requirements.txt ruff
    - name: Lint with ruff
      run: ruff check --output-format=github .

  code-formatting:
    runs-on: ubuntu-latest
    name: Code Format
    steps:
      - uses: actions/checkout@v4
      - name: Set up Python 3.11
        uses: actions/setup-python@v5
        with:
          python-version: "3.11"
      - name: Install ruff
        run: |
          python -m pip install --upgrade pip
          pip install ruff
      - name: Check with ruff
        run: ruff format --check .
        
  build:
    runs-on: ubuntu-latest
    name: Build the Python wheel
    steps:
      # For tags we assume the version in pyproject.toml is correct!
      - name: Checkout
        uses: actions/checkout@v4
      - name: Rewrite version for dev if not tag
        if: "!startsWith(github.ref, 'refs/tags/')"
        run: |
          perl -i -pe "s/version\s*=\s*\"(.*?)(\.rc.*|\.a.*|\.post.*)?\"/version=\"\1.dev0+${GITHUB_SHA::8}\"/" pyproject.toml
      - name: Note version
        run: |
          echo "PACKAGE_VERSION=$(tomlq '.project.version' pyproject.toml -r)" >> $GITHUB_ENV
      - name: Build localization MO files
        run: |
          installer/generate-mo.sh
      - name: Build Python wheels
        uses: RalfG/python-wheels-manylinux-build@v0.5.0-manylinux2014_x86_64
        with:
          python-versions: 'cp39-cp39 cp310-cp310 cp311-cp311 cp312-cp312'
          system-packages: 'gettext'
      - name: Upload wheels
        uses: actions/upload-artifact@v4
        with:
          name: wheels
          path: dist/*.whl
  deploy:
    if: startsWith(github.ref, 'refs/tags/')
    needs:
      - build
      - package-windows
      - package-mac
    runs-on: ubuntu-latest
    name: Deploy wheels to PyPI
    steps:
      - name: Download wheels
        uses: actions/download-artifact@v4
        with:
          name: wheels
      - name: Set up Python 3.11
        uses: actions/setup-python@v5
        with:
          python-version: 3.11
      - name: Upgrade pip
        run: |
          python -m pip install --upgrade pip
          pip install twine
      - name: Publish wheels to PyPI
        env:
          TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
          TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
        run: |
          twine upload *.whl
  deploy-to-artifact-storage:
    runs-on: [self-hosted, artifact-storage]
    name: Upload artifacts
    needs:
      - build
      - package-windows
      - package-mac
    concurrency:
      group: "${{ github.workflow }}-${{ github.ref }}"
    steps:
      - name: Clear
        run: |
          rm -rf "${{ github.workspace }}/artifacts" || true
          mkdir -p "${{ github.workspace }}/artifacts"
      - name: Download wheels
        id: artifact_wheel
        uses: actions/download-artifact@v4
        with:
          path: ${{ github.workspace }}/artifacts/whl
          name: wheels
      - name: Download Windows
        id: artifact_windows
        uses: actions/download-artifact@v4
        with:
          path: ${{ github.workspace }}/artifacts/win
          name: skytemple-windows-app-x64
      - name: Download MacOS Intel
        id: artifact_macos_x86
        uses: actions/download-artifact@v4
        with:
          path: ${{ github.workspace }}/artifacts/mac_x86
          name: skytemple-mac-dmg-x86_64
      - name: Copy over distributions
        run: |
          # Replace / in PR refs (123/merge -> 123-merge)
          REF=${GITHUB_REF_NAME/\//-}
          
          PATH_WHEEL=$(eval echo "${{ steps.artifact_wheel.outputs.download-path }}/*.whl")
          PATH_WIN=$(eval echo "${{ steps.artifact_windows.outputs.download-path }}/*.exe")
          PATH_MACOS=$(eval echo "${{ steps.artifact_macos_x86.outputs.download-path }}/*.dmg")
          
          mkdir -p /mnt/dist/randomizer
          cp -f "$PATH_WHEEL" "/mnt/dist/skytemple/skytemple-$REF.whl"
          cp -f "$PATH_WIN" "/mnt/dist/skytemple/skytemple-$REF.exe"
          cp -f "$PATH_MACOS" "/mnt/dist/skytemple/skytemple-x86-$REF.dmg"
  deploy-to-skytemple-dist:
    if: startsWith(github.ref, 'refs/tags/')
    runs-on: [self-hosted, skytemple-dist]
    name: Upload to Deposit Box
    needs:
      - build
      - package-windows
      - package-mac
    steps:
      - name: Clear
        run: |
          rm -rf "${{ github.workspace }}/artifacts" || true
          mkdir -p "${{ github.workspace }}/artifacts"
      - name: Download Windows
        id: artifact_windows
        uses: actions/download-artifact@v4
        with:
          path: ${{ github.workspace }}/artifacts/win
          name: skytemple-windows-app-x64
      - name: Download MacOS Intel
        id: artifact_macos_x86
        uses: actions/download-artifact@v4
        with:
          path: ${{ github.workspace }}/artifacts/mac_x86
          name: skytemple-mac-dmg-x86_64
      - name: Copy over distributions
        run: |
          PATH_WIN=$(eval echo "${{ steps.artifact_windows.outputs.download-path }}/*.exe")
          PATH_MACOS=$(eval echo "${{ steps.artifact_macos_x86.outputs.download-path }}/*.dmg")
          
          dist.py "$PATH_WIN" "skytemple/$GITHUB_REF_NAME/skytemple-$GITHUB_REF_NAME-win-install.exe"
          dist.py "$PATH_MACOS" "skytemple/$GITHUB_REF_NAME/skytemple-$GITHUB_REF_NAME-mac-install-x86-64.dmg"
  package-windows:
    # This is partly based on https://github.com/wingtk/gvsbuild/blob/main/.github/workflows/ci.yml
    runs-on: windows-2022
    name: Build and package for Windows
    steps:
      - name: Setup python
        uses: actions/setup-python@v5
        with:
          python-version: "3.11"
          architecture: "x64"
      - name: Cache GTK-Build
        id: cache-gtk-build
        uses: actions/cache@v4
        with:
          path: "C:\\gtk-build"
          key: win-gtk-build-2024-fork1-1
          restore-keys: |
            win-gtk-build-2023-fork1-1-
      - name: Checkout
        uses: actions/checkout@v4
      # Temporarily move the preinstalled git, it causes errors related to cygwin.
      - name: Move git binary
        run: |
          Move-Item "C:\Program Files\Git\usr\bin" "C:\Program Files\Git\usr\notbin"
          Move-Item "C:\Program Files\Git\bin" "C:\Program Files\Git\notbin"
      - name: Install gvsbuild
        run: |
          # gvsbuild has issues if the pipx install path has spaces in it. It may be installed
          # in Program Files (x86) by default, so let's just make sure it isn't.
          if (Test-Path "C:\Program Files (x86)\pipx") {
              Remove-Item -path "C:\Program Files (x86)\pipx" -recurse
          }
          
          # Make sure the pipx venv dir has no spaces
          New-Item -ItemType Directory -Force -Path C:\pipx_home
          $env:PIPX_HOME = "C:\pipx_home"
          echo "PIPX_HOME=C:\pipx_home" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
          New-Item -ItemType Directory -Force -Path C:\pipx_bin
          $env:PIPX_BIN_DIR = "C:\pipx_bin"
          echo "PIPX_BIN_DIR=C:\pipx_bin" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append

          $env:path = "C:\pipx_bin;" + $env:path
      
          python -m pip install --user pipx
          python -m pipx ensurepath
          pipx install "git+https://github.com/theCapypara/gvsbuild@pango-patch"
      - name: Build GTK and other libs
        run: |
          $env:path = "C:\pipx_bin;" + $env:path
          
          gvsbuild build --fast-build --ninja-opts -j2 --enable-gi --py-wheel gtk3 pygobject openssl gettext gtksourceview4 hicolor-icon-theme adwaita-icon-theme
      - name: Restore git binary
        run: |
          Move-Item "C:\Program Files\Git\usr\notbin" "C:\Program Files\Git\usr\bin"
          Move-Item "C:\Program Files\Git\notbin" "C:\Program Files\Git\bin"
      - name: Rewrite version for dev if not tag
        if: "!startsWith(github.ref, 'refs/tags/')"
        shell: sh
        run: |
          perl -i -pe "s/version\s*=\s*\"(.*?)(\.rc.*|\.a.*|\.post.*)?\"/version=\"\1.dev0+${GITHUB_SHA::8}\"/" pyproject.toml
          echo "IS_DEV_BUILD=1" >> $GITHUB_ENV
      - name: Note version
        run: |
          $env:path = "C:\pipx_bin;" + $env:path
          
          pipx install yq
          $env:PACKAGE_VERSION = tomlq.exe '.project.version' pyproject.toml -r
          echo "PACKAGE_VERSION=$env:PACKAGE_VERSION" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
      - name: Install SkyTemple
        run: |
          $ErrorActionPreference = "Stop"

          $env:path = "C:\gtk-build\gtk\x64\release\bin;" + $env:path
          $env:LIB = "C:\gtk-build\gtk\x64\release\lib;" + $env:LIB
          $env:INCLUDE = "C:\gtk-build\gtk\x64\release\include;C:\gtk-build\gtk\x64\release\include\cairo;C:\gtk-build\gtk\x64\release\include\glib-2.0;C:\gtk-build\gtk\x64\release\include\gobject-introspection-1.0;C:\gtk-build\gtk\x64\release\lib\glib-2.0\include;" + $env:INCLUDE

          cd installer
          
          pip debug --verbose  # print compatible tags etc. for debugging purposes

          # Package
          .\build-windows.ps1 $PACKAGE_VERSION

          
          if(!(Test-Path ".\dist\skytemple\skytemple.exe")){
              return 1
          }
          
          # Install latest skytemple-rust
          if ($env:IS_DEV_BUILD) {
              bash .\install-skytemple-rust.sh
          }
      - name: Create installer
        uses: joncloud/makensis-action@v3.7
        with:
          script-file: "installer/skytemple.nsi"
          arguments: "/DPRODUCT_VERSION=${{ env.PACKAGE_VERSION }}"

      - name: Upload artifacts
        uses: actions/upload-artifact@v4
        with:
          name: skytemple-windows-app-x64
          path: |
            installer/skytemple-*-install-*.exe

  package-mac:
    runs-on: macos-12
    name: Build and package for Mac OS (Intel)
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Rewrite version for dev if not tag
        if: "!startsWith(github.ref, 'refs/tags/')"
        run: |
          perl -i -pe "s/version\s*=\s*\"(.*?)(\.rc.*|\.a.*|\.post.*)?\"/version=\"\1.dev0+${GITHUB_SHA::8}\"/" pyproject.toml
          echo "IS_DEV_BUILD=1" >> $GITHUB_ENV
      - name: Note version
        run: |
          python3 -m venv .yq-venv
          . .yq-venv/bin/activate
          pip install yq
          echo "PACKAGE_VERSION=$(tomlq '.project.version' pyproject.toml -r)" >> $GITHUB_ENV
      - name: Install and package
        run: |
          # git is already installed.
          brew install enchant pygobject3 gtk+3 python@3.11 gtksourceview4 adwaita-icon-theme sdl12-compat sdl2 cmake
          
          python3.11 -m venv .venv
          . .venv/bin/activate
          pip3 debug --verbose  # print compatible tags etc. for debugging purposes
          
          pip3 install -U certifi


          # Install other dependencies and SkyTemple itself
          pip3 install py-desmume 'pyinstaller~=5.0'
          if [ -n "$IS_DEV_BUILD" ]; then
            IS_MACOS=1 installer/install-skytemple-rust.sh x86_64
          fi
          pip3 install -r requirements-mac-windows.txt
          # Generate MO localization files
          installer/generate-mo.sh
          pip3 install '.[eventserver]'
          if [ -n "$IS_DEV_BUILD" ]; then
            installer/install-skytemple-components-from-git.sh
          fi

          cd installer

          # Install themes
          curl https://skytemple.org/build_deps/Arc.zip -O
          unzip Arc.zip > /dev/null
          curl https://skytemple.org/build_deps/ZorinBlue.zip -O
          unzip ZorinBlue.zip > /dev/null

          # Download armips
          curl https://skytemple.org/build_deps/mac/armips -O
          chmod +x armips

          # Package
          ./build-mac.sh $PACKAGE_VERSION

      - name: Create installer
        run: |
          # See https://github.com/sindresorhus/create-dmg
          # create-dmg automatically generates an installer icon if imagemagick is installed
          brew install graphicsmagick imagemagick

          # This tool returns exit code 2 if the DMG was created but code signing failed for some reason
          npx create-dmg --dmg-title=SkyTemple installer/dist/SkyTemple.app installer || true

      - name: Upload .dmg
        uses: actions/upload-artifact@v4
        with:
          name: skytemple-mac-dmg-x86_64
          path: |
            installer/SkyTemple*.dmg