benchmark.yml

name: Benchmark

on:
  pull_request_target:
    branches: [ main ]
    types: [ opened ]
  issue_comment:
    types: [ created ]

jobs:
  check-command:
    runs-on: windows-2019
    permissions:
      pull-requests: write
    outputs:
      benchmark: ${{ steps.check.outputs.triggered }}
      ref: ${{ env.CUSTOM_REF }}
      repo: ${{ env.CUSTOM_REPO }}
    steps:
      - name: Print Event Name
        run: echo "${{ github.EVENT_NAME }}"

      - name: Check Benchmark Trigger
        if: ${{ github.EVENT_NAME != 'pull_request_target' }}
        uses: khan/pull-request-comment-trigger@master
        id: check
        with:
          trigger: '/benchmark'
          reaction: eyes
        env:
          GITHUB_TOKEN: '${{ secrets.GITHUB_TOKEN }}'

      - name: Find Branch
        uses: actions/github-script@v3
        id: find-branch
        with:
          script: |
            const request = {
              owner: context.repo.owner,
              repo: context.repo.repo,
              pull_number: context.issue.number
            }
            core.info(`Getting PR #${request.pull_number} from ${request.owner}/${request.repo}`)
            try {
              const result = await github.pulls.get(request)
              return result.data
            } catch (err) {
              core.setFailed(`Request failed with error ${err}`)
            }

      - name: Set Repository and Ref
        run: |
          $repository = '${{ fromJSON(steps.find-branch.outputs.result).head.repo.full_name }}'
          $ref = '${{ fromJSON(steps.find-branch.outputs.result).head.ref }}'
          echo "CUSTOM_REPO=$($repository)" >> $env:GITHUB_ENV
          echo "CUSTOM_REF=$($ref)" >> $env:GITHUB_ENV

  benchmark-build:
   if: ${{ needs.check-command.outputs.benchmark == 'true' || github.EVENT_NAME == 'pull_request_target' }}
   needs: check-command
   runs-on: windows-2019
   outputs:
     repo: ${{ needs.check-command.outputs.repo }}
     ref: ${{ needs.check-command.outputs.ref }}
     package_version: ${{ env.PACKAGE_VERSION }}
   steps:
     - uses: actions/checkout@v2
       with:
         repository: ${{ needs.check-command.outputs.repo }}
         ref: ${{ needs.check-command.outputs.ref }}
         submodules: 'true'

     - name: Ensure .NET 5 SDK
       uses: actions/setup-dotnet@v1
       with:
         dotnet-version: 5.0.302

     - name: Ensure .NET 6 SDK
       uses: actions/setup-dotnet@v1
       with:
         dotnet-version: 6.0.101

     - name: Install Build Dependencies
       run: |
         choco install meson
         choco install ninja
         choco install nasm
         echo "C:\Program Files\NASM" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
         echo "C:\Program Files\Meson" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
          
     - name: Set NuGet Version
       run: echo "PACKAGE_VERSION=1.0.0-benchmark.${{ GITHUB.RUN_NUMBER }}" >> $env:GITHUB_ENV
          
     - name: Build
       run: dotnet build -c Release
       working-directory: ./src/FileOnQ.Imaging.Heif
        
     - name: Pack
       run: dotnet pack -c Release -o ../../ /p:ContinuousIntegrationBuild=true /p:Version=${{ env.PACKAGE_VERSION }} /p:PackageVersion=${{ env.PACKAGE_VERSION }}
       working-directory: ./src/FileOnQ.Imaging.Heif
        
     - name: Upload NuGet Package
       uses: actions/upload-artifact@v2
       with:
         name: Benchmark-Packages
         path: |
           *.nupkg
           *.snupkg
    
  benchmark-run:
   if: ${{ needs.check-command.outputs.benchmark == 'true' || github.EVENT_NAME == 'pull_request_target' }}
   needs: benchmark-build
   runs-on: ${{ matrix.os }}
   outputs:
     repo: ${{ needs.benchmark-build.outputs.repo }}
     ref: ${{ needs.benchmark-build.outputs.ref }}
   strategy:
     matrix:
       include:
         - os: windows-2019
           framework: 'net48'
           benchmark: 'thumbnail'
         - os: windows-2019
           framework: 'net5.0'
           benchmark: 'thumbnail'
         - os: windows-2019
           framework: 'net6.0'
           benchmark: 'thumbnail'
         - os: windows-2019
           framework: 'net48'
           benchmark: 'primary'
         - os: windows-2019
           framework: 'net5.0'
           benchmark: 'primary'
         - os: windows-2019
           framework: 'net6.0'
           benchmark: 'primary'
   steps:
     - uses: actions/checkout@v2
       with:
         repository: ${{ needs.benchmark-build.outputs.repo }}
         ref: ${{ needs.benchmark-build.outputs.ref }}
         submodules: 'true'

     - name: Ensure .NET 5 SDK
       uses: actions/setup-dotnet@v1
       with:
         dotnet-version: 5.0.302

     - name: Ensure .NET 6 SDK
       uses: actions/setup-dotnet@v1
       with:
         dotnet-version: 6.0.101

     - name: Download Artifacts
       uses: actions/download-artifact@v2

     - name: Copy Nupkg
       shell: powershell
       run: Copy-Item Benchmark-Packages\* -Destination .
        
     - name: Update NuGet
       run: dotnet add package FileOnQ.Imaging.Heif --version ${{ needs.benchmark-build.outputs.package_version }}
       working-directory: ./benchmarks/tools

     - name: Benchmark
       run: dotnet run -c Release -f ${{ matrix.framework }} -b ${{ matrix.benchmark }}
       working-directory: ./benchmarks/tools

     - name: Upload Results
       uses: actions/upload-artifact@v2
       with:
         name: Benchmark-${{ matrix.benchmark }}.${{ matrix.framework }}
         path: ./benchmarks/tools/BenchmarkDotNet.Artifacts/results

  benchmark-compare:
    if: ${{ needs.check-command.outputs.benchmark == 'true' || github.EVENT_NAME == 'pull_request_target' }}
    needs: benchmark-run
    runs-on: windows-2019
    strategy:
      matrix:
        include:
          - framework: 'net48'
            name: '.NET Framework 4.8'
          - framework: 'net5.0'
            name: '.NET 5'
          - framework: 'net6.0'
            name: '.NET 6'
    steps:
      - name: Clone fileonq/imaging.heif
        uses: actions/checkout@v2
        with:
          repository: ${{ needs.benchmark-run.outputs.repo }}
          ref: ${{ needs.benchmark-run.outputs.ref }}
          path: fileonq.imaging.heif

      - name: Clone dotnet/performance
        uses: actions/checkout@v2
        with:
          repository: dotnet/performance
          path: dotnet.performance

      - name: Download Artifacts
        uses: actions/download-artifact@v2
        with:
          path: artifacts

      - name: Ensure .NET 5 SDK
        uses: actions/setup-dotnet@v1
        with:
          dotnet-version: 5.0.302

      - name: Build Compare Tool
        run: dotnet build
        working-directory: ./dotnet.performance/src/tools/ResultsComparer

      - name: Compare - thumbnail (${{ matrix.framework }})
        run: dotnet run  --base "../../../../fileonq.imaging.heif/benchmarks/results/thumbnail.${{ matrix.framework }}" --diff "../../../../artifacts/Benchmark-thumbnail.${{ matrix.framework }}" --threshold 10% --top 10 > thumbnail.${{ matrix.framework }}.output.txt
        working-directory: ./dotnet.performance/src/tools/ResultsComparer

      - name: Compare - primary (${{ matrix.framework }})
        run: dotnet run --base "../../../../fileonq.imaging.heif/benchmarks/results/primary.${{ matrix.framework }}" --diff "../../../../artifacts/Benchmark-primary.${{ matrix.framework }}" --threshold 10% --top 10 > primary.${{ matrix.framework }}.output.txt
        working-directory: ./dotnet.performance/src/tools/ResultsComparer

      - name: GitHub Comment - Comparison (${{ matrix.framework }})
        run: |
          $thumbnail = Get-Content "thumbnail.${{ matrix.framework }}.output.txt" -raw
          $primary = Get-Content "primary.${{ matrix.framework }}.output.txt" -raw
      
          $message = "## Benchmark Comparison - ${{ matrix.name }}`r`n"
          $message += "Benchmarking comparison between this Pull Request and the comitted values at benchmarks/results `r`n`r`n"
          $message += "thumbnail`r`n"
          $message += "\``\``\``ini `r`n $thumbnail `r`n \``\``\``"
          $message += "`r`n`r`nprimary`r`n"
          $message += "\``\``\``ini `r`n $primary `r`n \``\``\``"
          echo $message
          echo "PR_COMMENT_COMPARISON<<EOF" >> $env:GITHUB_ENV
          echo "$message" >> $env:GITHUB_ENV
          echo "EOF" >> $env:GITHUB_ENV
        working-directory: ./dotnet.performance/src/tools/ResultsComparer
      
      - name: GitHub Comment - Benchmark Results (${{ matrix.framework }})
        run: |
          $thumbnail = Get-Content "Benchmark-thumbnail.${{ matrix.framework }}/FileOnQ.Imaging.Heif.Benchmarks.Thumbnail-report-github.md" -raw
          $thumbnail = $thumbnail.replace("``", "\``")
      
          $primary = Get-Content "Benchmark-primary.${{ matrix.framework }}/FileOnQ.Imaging.Heif.Benchmarks.PrimaryImage-report-github.md" -raw
          $primary = $primary.replace("``", "\``")
      
          $message = "## Benchmark Results - ${{ matrix.name }}`r`n"
          $message += "<details><summary>thumbnail</summary><p>`r`n`r`n $thumbnail </p></details>`r`n"
          $message += "<details><summary>primary</summary><p>`r`n`r`n $primary </p></details>`r`n"
          echo $message
          echo "PR_COMMENT_RESULTS<<EOF" >> $env:GITHUB_ENV
          echo "$message" >> $env:GITHUB_ENV
          echo "EOF" >> $env:GITHUB_ENV
        working-directory: ./artifacts

      - name: Combine Message (${{ matrix.framework }})
        run: |
          $message = "${{ env.PR_COMMENT_COMPARISON }} \r\n\r\n ${{ env.PR_COMMENT_RESULTS }}"
          $message = $message.replace("\\\", "\``\``\``")
          echo $message > github-comment.${{ matrix.framework }}.md
        working-directory: ./artifacts

      - name: Upload Results
        uses: actions/upload-artifact@v2
        with:
          name: Benchmark-GitHub-Comment.${{ matrix.framework }}
          path: ./artifacts/github-comment.${{ matrix.framework }}.md

  benchmark-github-post:
    if: ${{ needs.check-command.outputs.benchmark == 'true' || github.EVENT_NAME == 'pull_request_target' }}
    needs: benchmark-compare
    runs-on: windows-2019
    steps:
      - name: Download Artifacts
        uses: actions/download-artifact@v2
        with:
          path: artifacts

      - name: Combine GitHub Comments
        run: |
          $net48 = Get-Content ./Benchmark-GitHub-Comment.net48/github-comment.net48.md -raw
          $net5 = Get-Content ./Benchmark-GitHub-Comment.net5.0/github-comment.net5.0.md -raw
          $net6 = Get-Content Benchmark-GitHub-Comment.net6.0/github-comment.net6.0.md -raw
          $message = "$net48`r`n`r`n$net5`r`n`r`n$net6"
          echo $message
          echo "GITHUB_COMMENT<<EOF" >> $env:GITHUB_ENV
          echo "$message" >> $env:GITHUB_ENV
          echo "EOF" >> $env:GITHUB_ENV
        working-directory: ./artifacts
        
      - name: GitHub Comment - Post
        uses: actions/github-script@v5
        with:
          script: |
            github.rest.issues.createComment({
              issue_number: context.issue.number,
              owner: context.repo.owner,
              repo: context.repo.repo,
              body: `${{ env.GITHUB_COMMENT }}`
            })