benchmarks.yaml

name: Benchmarks

on:
  push:
    branches: ["main"]
  pull_request:
    branches: ["main"]

jobs:

  benchmark_api:
    name: Benchmarks
    runs-on: ubuntu-22.04

    services:
      postgres:
        image: postgres
        env:
          POSTGRES_PASSWORD: postgres
          POSTGRES_DB: fractal_test
          FRACTAL_API_SUBMIT_RATE_LIMIT: 0
        options: >-
          --health-cmd pg_isready
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5
        ports:
          - 5432:5432

    steps:

     - uses: actions/checkout@v4
       with:
        fetch-depth: 1

     - name: Install poetry
       run: pipx install poetry==1.8.2

     - name: Set up Python 3.10
       uses: actions/setup-python@v5
       with:
        python-version: "3.10"
        cache: "poetry"

     - name: Install dependencies
       run: poetry install --with dev --without docs --no-interaction -E gunicorn -E postgres

     - name: Run Fractal with Gunicorn
       run: |
        export DB_ENGINE=postgres
        export POSTGRES_DB=fractal_test
        export POSTGRES_PASSWORD=postgres
        export POSTGRES_USER=postgres
        export POSTGRES_HOST=localhost
        export FRACTAL_RUNNER_WORKING_BASE_DIR=/tmp/artifacts
        export FRACTAL_TASKS_DIR=/tmp/FRACTAL_TASKS_DIR
        export JWT_SECRET_KEY=secret
        export JWT_EXPIRE_SECONDS=84600
        export FRACTAL_API_SUBMIT_RATE_LIMIT=0
        cd benchmarks/
        poetry run fractalctl set-db
        poetry run python ../scripts/populate_db/populate_db_script.py
        poetry run sh serve.sh

     - name: Benchmark
       run: |
        cd benchmarks/
        poetry run python api_bench.py

     - name: Print logs stderr
       run: cat benchmarks/fractal-server.err

     - name: Print logs stdout
       run: cat benchmarks/fractal-server.out


     - name: Add comment with preview
       uses: mshick/add-pr-comment@v2
       with:
         message-path: |
            benchmarks/bench_diff.md

     # commit on benchmark-api removing bench_diff.html
     - uses: stefanzweifel/git-auto-commit-action@v5
       if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
       with:
         commit_message: Update bench.html & bench.json
         branch: benchmark-api
         commit_options: '--no-verify'
         push_options: '--force'
         skip_checkout: true
         file_pattern: 'benchmarks/bench.html benchmarks/bench.json'

  benchmark_runner:
    if: false
    name: Benchmark Runner
    runs-on: ubuntu-22.04

    services:
      postgres:
        image: postgres
        env:
          POSTGRES_PASSWORD: postgres
          POSTGRES_DB: fractal_test
          FRACTAL_API_SUBMIT_RATE_LIMIT: 0
        options: >-
          --health-cmd pg_isready
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5
        ports:
          - 5432:5432

    steps:

     - uses: actions/checkout@v4
       with:
        fetch-depth: 1

     - name: Install poetry
       run: pipx install poetry==1.8.2

     - name: Set up Python 3.10
       uses: actions/setup-python@v5
       with:
        python-version: "3.10"
        cache: "poetry"

     - name: Install dependencies
       run: poetry install --with dev --without docs --no-interaction -E gunicorn -E postgres

     - name: Benchmark
       run: |
        cd benchmarks/runner/
        poetry run python benchmark_runner.py