build.yml

# SPDX-FileCopyrightText: 2014 MicroPython & CircuitPython contributors (https://github.com/adafruit/circuitpython/graphs/contributors)
#
# SPDX-License-Identifier: MIT

name: Build CI

on:
  push:
  pull_request:
  release:
    types: [published]
  check_suite:
    types: [rerequested]

concurrency:
  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
  cancel-in-progress: true

jobs:
  test:
    runs-on: ubuntu-20.04
    outputs:
      build-doc: ${{ steps.set-matrix.outputs.build-doc }}
      boards-arm: ${{ steps.set-matrix.outputs.boards-arm }}
      boards-riscv: ${{ steps.set-matrix.outputs.boards-riscv }}
      boards-espressif: ${{ steps.set-matrix.outputs.boards-espressif }}
      boards-aarch: ${{ steps.set-matrix.outputs.boards-aarch }}
    steps:
    - name: Dump GitHub context
      env:
        GITHUB_CONTEXT: ${{ toJson(github) }}
      run: echo "$GITHUB_CONTEXT"
    - uses: actions/checkout@v2.2.0
      with:
        submodules: false
        fetch-depth: 1
    - name: Set up Python 3
      uses: actions/setup-python@v2
      with:
        python-version: "3.x"
    - name: Get CP deps
      run: python tools/ci_fetch_deps.py test ${{ github.sha }}
    - name: CircuitPython version
      run: |
        tools/describe || git log --parents HEAD~4..
        echo >>$GITHUB_ENV CP_VERSION=$(tools/describe)
    - name: Install dependencies
      run: |
        sudo apt-get update
        sudo apt-get install -y eatmydata
        sudo eatmydata apt-get install -y gettext gcc-aarch64-linux-gnu mingw-w64
        pip install -r requirements-ci.txt -r requirements-dev.txt
    - name: Versions
      run: |
        gcc --version
        python3 --version
    - name: Duplicate USB VID/PID Check
      run: python3 -u -m tools.ci_check_duplicate_usb_vid_pid
    - name: Build mpy-cross
      run: make -C mpy-cross -j2
    - name: Build unix port
      run: |
        make -C ports/unix VARIANT=coverage -j2
    - name: Test all
      run: MICROPY_CPYTHON3=python3.8 MICROPY_MICROPYTHON=../ports/unix/micropython-coverage ./run-tests.py -j1
      working-directory: tests
    - name: Print failure info
      run: MICROPY_CPYTHON3=python3.8 MICROPY_MICROPYTHON=../ports/unix/micropython-coverage ./run-tests.py -j1 --print-failures
      if: failure()
      working-directory: tests
    - name: Native Tests
      run: MICROPY_CPYTHON3=python3.8 MICROPY_MICROPYTHON=../ports/unix/micropython-coverage ./run-tests.py -j1 --emit native
      working-directory: tests
    - name: mpy Tests
      run: MICROPY_CPYTHON3=python3.8 MICROPY_MICROPYTHON=../ports/unix/micropython-coverage ./run-tests.py -j1 --via-mpy -d basics float micropython
      working-directory: tests
    - name: Native mpy Tests
      run: MICROPY_CPYTHON3=python3.8 MICROPY_MICROPYTHON=../ports/unix/micropython-coverage ./run-tests.py -j1 --via-mpy --emit native -d basics float micropython
      working-directory: tests
    - name: Build native modules
      run: |
        make -C examples/natmod/features1
        make -C examples/natmod/features2
        make -C examples/natmod/btree
        make -C examples/natmod/framebuf
        make -C examples/natmod/uheapq
        make -C examples/natmod/urandom
        make -C examples/natmod/ure
        make -C examples/natmod/uzlib
    - name: Test native modules
      run: MICROPY_CPYTHON3=python3.8 MICROPY_MICROPYTHON=../ports/unix/micropython-coverage ./run-natmodtests.py extmod/{btree*,framebuf*,uheapq*,ure*,uzlib*}.py
      working-directory: tests
    - name: Build mpy-cross.static-aarch64
      run: make -C mpy-cross -j2 -f Makefile.static-aarch64
    - uses: actions/upload-artifact@v2
      with:
        name: mpy-cross.static-aarch64
        path: mpy-cross/mpy-cross.static-aarch64
    - name: Build mpy-cross.static-raspbian
      run: make -C mpy-cross -j2 -f Makefile.static-raspbian
    - uses: actions/upload-artifact@v2
      with:
        name: mpy-cross.static-raspbian
        path: mpy-cross/mpy-cross.static-raspbian
    - name: Build mpy-cross.static
      run: make -C mpy-cross -j2 -f Makefile.static
    - uses: actions/upload-artifact@v2
      with:
        name: mpy-cross.static-amd64-linux
        path: mpy-cross/mpy-cross.static
    - name: Build mpy-cross.static-mingw
      run: make -C mpy-cross -j2 -f Makefile.static-mingw
    - uses: actions/upload-artifact@v2
      with:
        name: mpy-cross.static-x64-windows
        path: mpy-cross/mpy-cross.static.exe
    - name: Upload mpy-cross builds to S3
      if: (github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') || (github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested'))
      env:
        AWS_PAGER: ''
        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
      run: |
        [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp mpy-cross/mpy-cross.static-aarch64 s3://adafruit-circuit-python/bin/mpy-cross/mpy-cross.static-aarch64-${{ env.CP_VERSION }} --no-progress --region us-east-1
        [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp mpy-cross/mpy-cross.static-raspbian s3://adafruit-circuit-python/bin/mpy-cross/mpy-cross.static-raspbian-${{ env.CP_VERSION }} --no-progress --region us-east-1
        [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp mpy-cross/mpy-cross.static s3://adafruit-circuit-python/bin/mpy-cross/mpy-cross.static-amd64-linux-${{ env.CP_VERSION }} --no-progress --region us-east-1
        [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp mpy-cross/mpy-cross.static.exe s3://adafruit-circuit-python/bin/mpy-cross/mpy-cross.static-x64-windows-${{ env.CP_VERSION }}.exe --no-progress --region us-east-1
    - name: "Get changes"
      if: github.event_name == 'pull_request'
      uses: dorny/paths-filter@v2
      id: filter
      with:
        list-files: json
        filters: |
          changed:
            - '**'
    - name: "Set matrix"
      id: set-matrix
      working-directory: tools
      env:
        CHANGED_FILES: ${{ steps.filter.outputs.changed_files }}
      run: python3 -u ci_set_matrix.py


  mpy-cross-mac:
    runs-on: macos-11
    steps:
    - name: Dump GitHub context
      env:
        GITHUB_CONTEXT: ${{ toJson(github) }}
      run: echo "$GITHUB_CONTEXT"
    - uses: actions/checkout@v2.2.0
      with:
        submodules: false
        fetch-depth: 1
    - name: Set up Python 3
      uses: actions/setup-python@v2
      with:
        python-version: "3.x"
    - name: Get CP deps
      run: python tools/ci_fetch_deps.py mpy-cross-mac ${{ github.sha }}
    - name: CircuitPython version
      run: |
        tools/describe || git log --parents HEAD~4..
        echo >>$GITHUB_ENV CP_VERSION=$(tools/describe)
    - name: Install dependencies
      run: |
        brew install gettext
        echo >>$GITHUB_PATH /usr/local/opt/gettext/bin
    - name: Versions
      run: |
        gcc --version
        python3 --version
        msgfmt --version
    - name: Build mpy-cross
      run: make -C mpy-cross -j2
    - uses: actions/upload-artifact@v2
      with:
        name: mpy-cross-macos-11-x64
        path: mpy-cross/mpy-cross
    - name: Build mpy-cross (arm64)
      run: make -C mpy-cross -j2 -f Makefile.m1 V=2
    - uses: actions/upload-artifact@v2
      with:
        name: mpy-cross-macos-11-arm64
        path: mpy-cross/mpy-cross-arm64
    - name: Make universal binary
      run: lipo -create -output mpy-cross-macos-universal mpy-cross/mpy-cross mpy-cross/mpy-cross-arm64
    - uses: actions/upload-artifact@v2
      with:
        name: mpy-cross-macos-11-universal
        path: mpy-cross-macos-universal
    - name: Upload mpy-cross build to S3
      run: |
        [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp mpy-cross-macos-universal s3://adafruit-circuit-python/bin/mpy-cross/mpy-cross-macos-11-${{ env.CP_VERSION }}-universal --no-progress --region us-east-1
        [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp mpy-cross/mpy-cross-arm64 s3://adafruit-circuit-python/bin/mpy-cross/mpy-cross-macos-11-${{ env.CP_VERSION }}-arm64 --no-progress --region us-east-1
        [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp mpy-cross/mpy-cross s3://adafruit-circuit-python/bin/mpy-cross/mpy-cross-macos-11-${{ env.CP_VERSION }}-x64 --no-progress --region us-east-1
      env:
        AWS_PAGER: ''
        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
      if: (github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') || (github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested'))


  build-doc:
    runs-on: ubuntu-20.04
    needs: test
    if: ${{ needs.test.outputs.build-doc == 'True' }}
    steps:
    - uses: actions/checkout@v2.2.0
      with:
        submodules: false
        fetch-depth: 1
    - name: Get CP deps
      run: python tools/ci_fetch_deps.py docs ${{ github.sha }}
    - name: CircuitPython version
      run: |
        tools/describe || git log --parents HEAD~4..
        echo >>$GITHUB_ENV CP_VERSION=$(tools/describe)
    - name: Set up Python 3
      uses: actions/setup-python@v2
      with:
        python-version: "3.x"
    - name: Install dependencies
      run: |
        sudo apt-get update
        sudo apt-get install -y eatmydata
        sudo eatmydata apt-get install -y latexmk librsvg2-bin texlive-fonts-recommended texlive-latex-recommended texlive-latex-extra
        pip install -r requirements-ci.txt -r requirements-doc.txt
    - name: Build and Validate Stubs
      run: make check-stubs -j2
    - uses: actions/upload-artifact@v2
      with:
        name: stubs
        path: circuitpython-stubs/dist/*
    - name: Test Documentation Build (HTML)
      run: sphinx-build -E -W -b html -D version=${{ env.CP_VERSION }} -D release=${{ env.CP_VERSION }} . _build/html
    - uses: actions/upload-artifact@v2
      with:
        name: docs
        path: _build/html
    - name: Test Documentation Build (LaTeX/PDF)
      run: |
        make latexpdf
    - uses: actions/upload-artifact@v2
      with:
        name: docs
        path: _build/latex
    - name: Upload stubs to S3
      if: (github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') || (github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested'))
      env:
        AWS_PAGER: ''
        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
      run: |
        zip -9r circuitpython-stubs.zip circuitpython-stubs
        [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp circuitpython-stubs/dist/*.tar.gz s3://adafruit-circuit-python/bin/stubs/circuitpython-stubs-${{ env.CP_VERSION }}.zip --no-progress --region us-east-1
    - name: Upload stubs to PyPi
      if: github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested')
      env:
        TWINE_USERNAME: ${{ secrets.pypi_username }}
        TWINE_PASSWORD: ${{ secrets.pypi_password }}
      run: |
        # setup.py sdist was run by 'make stubs'
        [ -z "$TWINE_USERNAME" ] || echo "Uploading dev release to PyPi"
        [ -z "$TWINE_USERNAME" ] || twine upload circuitpython-stubs/dist/*


  build-arm:
    runs-on: ubuntu-20.04
    needs: test
    strategy:
      fail-fast: false
      matrix:
        board: ${{ fromJSON(needs.test.outputs.boards-arm) }}
    if: ${{ needs.test.outputs.boards-arm != '[]' }}
    steps:
    - name: Set up Python 3
      uses: actions/setup-python@v2
      with:
        python-version: "3.x"
    - uses: actions/checkout@v2.2.0
      with:
        submodules: false
        fetch-depth: 1
    - name: Get CP deps
      run: python tools/ci_fetch_deps.py ${{ matrix.board }} ${{ github.sha }}
    - uses: carlosperate/arm-none-eabi-gcc-action@v1
      with:
        release: '10-2020-q4'
    - name: Install dependencies
      run: |
        sudo apt-get install -y gettext
        pip install -r requirements-ci.txt -r requirements-dev.txt
    - name: Versions
      run: |
        gcc --version
        arm-none-eabi-gcc --version
        python3 --version
    - name: mpy-cross
      run: make -C mpy-cross -j2
    - name: Setup build failure matcher
      run: echo "::add-matcher::$GITHUB_WORKSPACE/.github/workflows/match-build-fail.json"
    - name: build
      run: python3 -u build_release_files.py
      working-directory: tools
      env:
        BOARDS: ${{ matrix.board }}
    - uses: actions/upload-artifact@v2
      with:
        name: ${{ matrix.board }}
        path: bin/${{ matrix.board }}
    - name: Upload to S3
      run: "[ -z \"$AWS_ACCESS_KEY_ID\" ] || aws s3 cp bin/ s3://adafruit-circuit-python/bin/ --recursive --no-progress --region us-east-1"
      env:
        AWS_PAGER: ''
        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
      if: (github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') || (github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested'))


  build-riscv:
    runs-on: ubuntu-20.04
    needs: test
    strategy:
      fail-fast: false
      matrix:
        board: ${{ fromJSON(needs.test.outputs.boards-riscv) }}
    if: ${{ needs.test.outputs.boards-riscv != '[]' }}
    steps:
    - name: Set up Python 3
      uses: actions/setup-python@v2
      with:
        python-version: "3.x"
    - uses: actions/checkout@v2.2.0
      with:
        submodules: false
        fetch-depth: 1
    - name: Get CP deps
      run: python tools/ci_fetch_deps.py ${{ matrix.board }} ${{ github.sha }}
    - name: Install dependencies
      run: |
        sudo apt-get install -y gettext
        pip install -r requirements-ci.txt -r requirements-dev.txt
        wget https://static.dev.sifive.com/dev-tools/riscv64-unknown-elf-gcc-8.3.0-2019.08.0-x86_64-linux-centos6.tar.gz
        sudo tar -C /usr --strip-components=1 -xaf riscv64-unknown-elf-gcc-8.3.0-2019.08.0-x86_64-linux-centos6.tar.gz
    - name: Versions
      run: |
        gcc --version
        riscv64-unknown-elf-gcc --version
        python3 --version
    - name: mpy-cross
      run: make -C mpy-cross -j2
    - name: Setup build failure matcher
      run: echo "::add-matcher::$GITHUB_WORKSPACE/.github/workflows/match-build-fail.json"
    - name: build
      run: python3 -u build_release_files.py
      working-directory: tools
      env:
        BOARDS: ${{ matrix.board }}
    - uses: actions/upload-artifact@v2
      with:
        name: ${{ matrix.board }}
        path: bin/${{ matrix.board }}
    - name: Upload to S3
      run: "[ -z \"$AWS_ACCESS_KEY_ID\" ] || aws s3 cp bin/ s3://adafruit-circuit-python/bin/ --recursive --no-progress --region us-east-1"
      env:
        AWS_PAGER: ''
        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
      if: (github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') || (github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested'))


  build-espressif:
    runs-on: ubuntu-20.04
    needs: test
    strategy:
      fail-fast: false
      matrix:
        board: ${{ fromJSON(needs.test.outputs.boards-espressif) }}
    if: ${{ needs.test.outputs.boards-espressif != '[]' }}
    steps:
    - name: Set up Python 3
      id: py3
      uses: actions/setup-python@v4
      with:
        python-version: "3.x"
    - uses: actions/checkout@v2.2.0
      with:
        submodules: false
        fetch-depth: 1
    - name: Get CP deps
      run: python tools/ci_fetch_deps.py ${{ matrix.board }} ${{ github.sha }}
    - name: CircuitPython version
      run: |
        tools/describe || git log --parents HEAD~4..
        echo >>$GITHUB_ENV CP_VERSION=$(tools/describe)
    - uses: actions/cache@v2
      name: Fetch IDF tool cache
      id: idf-cache
      with:
        path: ${{ github.workspace }}/.idf_tools
        key: ${{ runner.os }}-idf-tools-${{ hashFiles('.git/modules/ports/espressif/esp-idf/HEAD') }}-${{ steps.py3.outputs.python-path }}-20220404
    - name: Clone IDF submodules
      run: |
        (cd $IDF_PATH && git submodule update --init)
      env:
        IDF_PATH: ${{ github.workspace }}/ports/espressif/esp-idf
    - name: Install IDF tools
      run: |
        $IDF_PATH/tools/idf_tools.py --non-interactive install required
        $IDF_PATH/tools/idf_tools.py --non-interactive install cmake
        $IDF_PATH/tools/idf_tools.py --non-interactive install-python-env
        rm -rf $IDF_TOOLS_PATH/dist
      env:
        IDF_PATH: ${{ github.workspace }}/ports/espressif/esp-idf
        IDF_TOOLS_PATH: ${{ github.workspace }}/.idf_tools
    - name: Install dependencies
      run: |
        source $IDF_PATH/export.sh
        sudo apt-get install -y gettext ninja-build
        pip install -r requirements-ci.txt -r requirements-dev.txt
      env:
        IDF_PATH: ${{ github.workspace }}/ports/espressif/esp-idf
        IDF_TOOLS_PATH: ${{ github.workspace }}/.idf_tools
    - name: Versions
      run: |
        source $IDF_PATH/export.sh
        gcc --version
        xtensa-esp32s2-elf-gcc --version
        python3 --version
        ninja --version
        cmake --version
      shell: bash
      env:
        IDF_PATH: ${{ github.workspace }}/ports/espressif/esp-idf
        IDF_TOOLS_PATH: ${{ github.workspace }}/.idf_tools
    - name: mpy-cross
      run: make -C mpy-cross -j2
    - name: Setup build failure matcher
      run: echo "::add-matcher::$GITHUB_WORKSPACE/.github/workflows/match-build-fail.json"
    - name: build
      run: |
        source $IDF_PATH/export.sh
        python3 -u build_release_files.py
      working-directory: tools
      shell: bash
      env:
        IDF_PATH: ${{ github.workspace }}/ports/espressif/esp-idf
        IDF_TOOLS_PATH: ${{ github.workspace }}/.idf_tools
        BOARDS: ${{ matrix.board }}
    - uses: actions/upload-artifact@v2
      with:
        name: ${{ matrix.board }}
        path: bin/${{ matrix.board }}
    - name: Upload to S3
      run: "[ -z \"$AWS_ACCESS_KEY_ID\" ] || aws s3 cp bin/ s3://adafruit-circuit-python/bin/ --recursive --no-progress --region us-east-1"
      env:
        AWS_PAGER: ''
        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
      if: (github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') || (github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested'))

  build-aarch:
    runs-on: ubuntu-20.04
    needs: test
    strategy:
      fail-fast: false
      matrix:
        board: ${{ fromJSON(needs.test.outputs.boards-aarch) }}
    if: ${{ needs.test.outputs.boards-aarch != '[]' }}
    steps:
    - name: Set up Python 3
      uses: actions/setup-python@v2
      with:
        python-version: "3.x"
    - uses: actions/checkout@v2.2.0
      with:
        submodules: false
        fetch-depth: 1
    - name: Get CP deps
      run: python tools/ci_fetch_deps.py ${{ matrix.board }} ${{ github.sha }}
    - name: Install dependencies
      run: |
        sudo apt-get install -y gettext mtools
        pip install -r requirements-ci.txt -r requirements-dev.txt
        wget --no-verbose https://adafruit-circuit-python.s3.amazonaws.com/gcc-arm-10.3-2021.07-x86_64-aarch64-none-elf.tar.xz
        sudo tar -C /usr --strip-components=1 -xaf gcc-arm-10.3-2021.07-x86_64-aarch64-none-elf.tar.xz
    - uses: carlosperate/arm-none-eabi-gcc-action@v1
      with:
        release: '10-2020-q4'
    - name: Install mkfs.fat
      run: |
        wget https://github.com/dosfstools/dosfstools/releases/download/v4.2/dosfstools-4.2.tar.gz
        tar -xaf dosfstools-4.2.tar.gz
        cd dosfstools-4.2
        ./configure
        make -j 2
        cd src
        echo >>$GITHUB_PATH $(pwd)
    - name: Versions
      run: |
        gcc --version
        aarch64-none-elf-gcc --version
        arm-none-eabi-gcc --version
        python3 --version
        mkfs.fat --version || true
    - name: mpy-cross
      run: make -C mpy-cross -j2
    - name: Setup build failure matcher
      run: echo "::add-matcher::$GITHUB_WORKSPACE/.github/workflows/match-build-fail.json"
    - name: build
      run: python3 -u build_release_files.py
      working-directory: tools
      env:
        BOARDS: ${{ matrix.board }}
    - uses: actions/upload-artifact@v2
      with:
        name: ${{ matrix.board }}
        path: bin/${{ matrix.board }}
    - name: Upload to S3
      run: "[ -z \"$AWS_ACCESS_KEY_ID\" ] || aws s3 cp bin/ s3://adafruit-circuit-python/bin/ --recursive --no-progress --region us-east-1"
      env:
        AWS_PAGER: ''
        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
      if: (github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') || (github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested'))