test-and-deploy.yml

name: Test, Build and Deploy Images

env:
  DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}

on:
    workflow_dispatch:
    pull_request:
    push:
    schedule:
    # Run every 6 days to keep our caches alive
    - cron: '0 0 */6 * *'

jobs:
  python-tests:
    runs-on: ubuntu-latest

    env:
      python-version: 3.11 # Our base image has Python 3.11

    steps:
    - name: Checkout
      uses: actions/checkout@v3

    - name: Set up Python ${{ env.python-version }}
      uses: actions/setup-python@v4
      with:
        python-version: ${{ env.python-version }}

    - name: Install dependencies
      run: |
        sudo apt install shellcheck
        python -m pip install --upgrade pip
        pip install pipenv
        pipenv install poetry && pipenv run poetry install
        echo "installing mavlink-router"
        pipenv run ./core/tools/mavlink_router/bootstrap.sh
        echo "installing mavp2p"
        pipenv run ./core/tools/mavp2p/bootstrap.sh
        echo "installing ardupilot_tools"
        pipenv run ./core/tools/ardupilot_tools/bootstrap.sh
        pipenv run python ./core/libs/bridges/setup.py install
        pipenv run python ./core/libs/commonwealth/setup.py install
        rm -rf core/libs/bridges/build
        rm -rf core/libs/commonwealth/build

    - name: Run tests
      run: |
        .hooks/pre-push

    - name: Upload coverage
      uses: actions/upload-artifact@v3
      with:
        name: coverage
        path: htmlcov


  frontend-tests:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v3
        with:
          submodules: recursive

      - uses: oven-sh/setup-bun@v1
        with:
          bun-version: 1.0.3

      - name: Bun install
        run: bun install --cwd ./core/frontend

      - name: Bun lint
        run: bun --cwd ./core/frontend lint

      - name: Bun build
        run: bun run --cwd ./core/frontend build

  deploy-docker-images:
    runs-on: ubuntu-latest
    needs: [python-tests, frontend-tests]
    strategy:
      matrix:
        docker: [bootstrap, core]
        project: [blueos]
        platforms: ["linux/arm/v7,linux/arm64/v8,linux/amd64"]
    steps:
      - name: Checkout
        uses: actions/checkout@v3
        with:
          fetch-depth: 0 #Number of commits to fetch. 0 indicates all history for all branches and tags.
          submodules: recursive

      - name: Prepare
        id: prepare
        run: |
          # Deploy image with the name of the branch, if the build is a git tag, replace tag with the tag name.
          # If git tag matches semver, append latest tag to the push.
          DOCKER_IMAGE=${DOCKER_USERNAME:-bluerobotics}/${{ matrix.project }}-${{ matrix.docker }}
          VERSION=${GITHUB_REF##*/}

          if [[ $GITHUB_REF == refs/tags/* ]]; then
            VERSION=${GITHUB_REF#refs/tags/}
          fi

          TAGS="--tag ${DOCKER_IMAGE}:${VERSION}"
          if [[ $VERSION =~ ^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$ ]]; then
            TAGS="$TAGS --tag ${DOCKER_IMAGE}:latest"
          fi

          echo "docker_image=${DOCKER_IMAGE}" >> $GITHUB_OUTPUT
          echo "version=${VERSION}" >> $GITHUB_OUTPUT
          echo "buildx_args=\
            --build-arg GIT_DESCRIBE_TAGS=$(git describe --tags --long --always) \
            --build-arg VITE_APP_GIT_DESCRIBE=$(git describe --long --always --dirty --all) \
            --cache-from 'type=local,src=/tmp/.buildx-cache' \
            --cache-to 'type=local,dest=/tmp/.buildx-cache' \
            ${TAGS} \
            --file ${{ matrix.docker }}/Dockerfile ./${{ matrix.docker }}" >> $GITHUB_OUTPUT

      - name: Set up QEMU
        uses: docker/setup-qemu-action@v3
        with:
          platforms: all

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3
        with:
          version: latest

      - name: Cache Docker layers
        uses: actions/cache@v3
        id: cache
        with:
          path: /tmp/.buildx-cache
          key: ${{ runner.os }}-buildx-${{ matrix.docker }}-${{ hashFiles(format('{0}/Dockerfile', matrix.docker)) }}
          restore-keys: |
            ${{ runner.os }}-buildx-${{ matrix.docker }}-${{ hashFiles(format('{0}/Dockerfile', matrix.docker)) }}
            ${{ runner.os }}-buildx-${{ matrix.docker }}

      - name: Docker Buildx (build)
        run: |
          # Pull latest version of image to help with build speed
          for platform in $(echo ${{ matrix.platforms }} | tr ',' '\n'); do
            docker pull --platform ${platform} ${DOCKER_USERNAME:-bluerobotics}/${{ matrix.project }}-${{ matrix.docker }}:master || true
          done
          docker buildx build \
            --output "type=image,push=false" \
            --platform ${{ matrix.platforms }} \
            ${{ steps.prepare.outputs.buildx_args }}

      - name: Check core size
        if: ${{ matrix.docker == 'core' }}
        run: |
          # Check if the image size is lower than our limit
          docker image list
          IMAGE_ID=$(docker images -q ${DOCKER_USERNAME:-bluerobotics}/${{ matrix.project }} | head -n 1)
          LIMIT_SIZE_MB=700
          IMAGE_SIZE_MB=$(( $(docker inspect $IMAGE_ID --format {{.Size}})/(2**20) ))
          echo "Core size is: $IMAGE_SIZE_MB MB"
          ((IMAGE_SIZE_MB < LIMIT_SIZE_MB))

      - name: Login to DockerHub
        if: success() && github.event_name != 'pull_request'
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKER_USERNAME }}
          password: ${{ secrets.DOCKER_PASSWORD }}

      - name: Docker Buildx (push)
        if: success() && github.event_name != 'pull_request'
        run: |
          docker buildx build \
            --output "type=image,push=true" \
            --platform ${{ matrix.platforms }} \
            ${{ steps.prepare.outputs.buildx_args }}

      - name: Inspect image
        if: always() && github.event_name != 'pull_request'
        run: |
          docker buildx imagetools \
            inspect ${{ steps.prepare.outputs.docker_image }}:${{ steps.prepare.outputs.version }}

      - name: Create image artifact
        if: success() && matrix.docker == 'core'
        run: |
          DOCKER_IMAGE=${DOCKER_USERNAME:-bluerobotics}/${{ matrix.project }}-${{ matrix.docker }}
          GIT_HASH_SHORT=$(git rev-parse --short "$GITHUB_SHA")
          docker buildx build \
            ${{ steps.prepare.outputs.buildx_args }} \
            --platform "linux/arm64/v8"  \
            --tag ${DOCKER_IMAGE}:${GIT_HASH_SHORT} \
            --output "type=docker,dest=BlueOs-core-docker-image-${GIT_HASH_SHORT}-arm64-v8.tar" \

      - name: Upload artifact arm64-v8
        uses: actions/upload-artifact@v3
        if: success() && matrix.docker == 'core'
        with:
          name: BlueOS-core-docker-image-arm64-v8
          path: '*arm64-v8.tar'

      - name: Create image artifact
        if: success() && matrix.docker == 'core'
        run: |
          DOCKER_IMAGE=${DOCKER_USERNAME:-bluerobotics}/${{ matrix.project }}-${{ matrix.docker }}
          GIT_HASH_SHORT=$(git rev-parse --short "$GITHUB_SHA")
          docker buildx build \
            ${{ steps.prepare.outputs.buildx_args }} \
            --platform "linux/arm/v7"  \
            --tag ${DOCKER_IMAGE}:${GIT_HASH_SHORT} \
            --output "type=docker,dest=BlueOs-core-docker-image-${GIT_HASH_SHORT}-arm-v7.tar" \

      - name: Upload artifact arm-v7
        uses: actions/upload-artifact@v3
        if: success() && matrix.docker == 'core'
        with:
          name: BlueOS-core-docker-image-arm-v7
          path: '*arm-v7.tar'

      - name: Create image artifact
        if: success() && matrix.docker == 'core'
        run: |
          DOCKER_IMAGE=${DOCKER_USERNAME:-bluerobotics}/${{ matrix.project }}-${{ matrix.docker }}
          GIT_HASH_SHORT=$(git rev-parse --short "$GITHUB_SHA")
          docker buildx build \
            ${{ steps.prepare.outputs.buildx_args }} \
            --platform "linux/amd64"  \
            --tag ${DOCKER_IMAGE}:${GIT_HASH_SHORT} \
            --output "type=docker,dest=BlueOs-core-docker-image-${GIT_HASH_SHORT}-amd64.tar" \

      - name: Upload artifact amd64
        uses: actions/upload-artifact@v3
        if: success() && matrix.docker == 'core'
        with:
          name: BlueOS-core-docker-image-amd64
          path: '*amd64.tar'

      - name: Upload docker image for release
        uses: svenstaro/upload-release-action@v2
        if: startsWith(github.ref, 'refs/tags/') && success() && matrix.docker == 'core'
        with:
          repo_token: ${{ secrets.GITHUB_TOKEN }}
          file: '*.tar'
          tag: ${{ github.ref }}
          overwrite: true
          prerelease: true
          file_glob: true

  deploy-raspberry-image:
    needs: deploy-docker-images
    if: github.event_name != 'pull_request'
    timeout-minutes: 180 # Detect if it gets into an infinite loop or some unexpected state
    runs-on: blueos-ci
    # The runner for this job is a Raspberry Pi OS Bullseye.
    # Just install docker (curl -sSL https://get.docker.com/ | sh) and follow the instructions for setting up a new runner in
    # https://github.com/bluerobotics/BlueOS-docker/settings/actions/runners/new

    strategy:
      matrix:
        platforms: ["linux/arm/v7"]
    steps:
      - name: Install git
        run: sudo apt install -y git

      - name: Checkout
        uses: actions/checkout@v3
        with:
          submodules: recursive


      # We use our own pimod as upstream doesn't provide armv7 images
      - name: Pimod Build
        run: |
          VERSION=$GITHUB_REPOSITORY
          VERSION=${VERSION:-master}
          wget https://raw.githubusercontent.com//Nature40/pimod/master/pimod.sh && chmod +x pimod.sh
          docker run --rm --privileged \
            -v $PWD:/files \
            -e PATH=/pimod:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \
            -e GITHUB_REPOSITORY=$GITHUB_REPOSITORY \
            -e VERSION=$GITHUB_REF_NAME \
            --workdir=/files \
            --platform ${{ matrix.platforms }} nature40/pimod:latest pimod.sh deploy/pimod/blueos.Pifile

      - name: Add /boot additions
        run: |
          sudo apt-get update && sudo apt-get install -y parted kpartx
          # Create mount point if it doesn't exist
          sudo mkdir -p /mnt/piboot
          # Map the image file
          LOOP_DEVICE=$(sudo kpartx -avs deploy/pimod/blueos.img | sed -E 's/.*(loop[0-9]+)p[0-9]+.*/\1/g' | head -1)
          # Mount the boot partition
          sudo mount "/dev/mapper/${LOOP_DEVICE}p1" /mnt/piboot
          # Create ssh and userconf files
          sudo touch "${MOUNT_POINT}/ssh"
          echo "pi:$y$j9T$9ZU7FXeU7PVHm49rB5Hbe.$MHZMLJfNfXBFbYgGwg/TNjADIRbna.Epxzd/MrqPJM9" | sudo tee /mnt/piboot/userconf > /dev/null
          # Unmount and cleanup
          sudo umount /mnt/piboot
          sudo kpartx -d deploy/pimod/blueos.img
          echo "Boot partition updated successfully."

      # TODO: add GITHUB_REF_NAME after https://github.com/actions/upload-artifact/issues/231 is fixed
      # name: blueos-${{ env.GITHUB_REF_NAME }}.zip
      - name: Zip image
        if: startsWith(github.ref, 'refs/tags/')
        run: |
          sudo apt install zip
          zip BlueOS-raspberry.zip deploy/pimod/blueos.img

      - name: Upload artifact
        uses: actions/upload-artifact@v3
        timeout-minutes: 120
        with:
          # We use the .img to avoid a zip of a zip
          name: BlueOS-raspberry.zip
          path: deploy/pimod/blueos.img
          if-no-files-found: error
          retention-days: 7

      - name: Upload raspberry image for release
        uses: svenstaro/upload-release-action@v2
        if: startsWith(github.ref, 'refs/tags/')
        with:
          repo_token: ${{ secrets.GITHUB_TOKEN }}
          file: BlueOS-raspberry.zip
          tag: ${{ github.ref }}
          overwrite: true
          prerelease: true

      # This is required because docker has root permissions, which means the runner is unable to clear this cache normally
      - name: Cleanup
        if: ${{ always() }}
        run: sudo rm -rf .cache