Skip to content

Benchmark

Benchmark #85

Workflow file for this run

name: Benchmark
on:
pull_request:
branches: ['*']
paths:
- .github/workflows/benchmark.yml
schedule:
- cron: '0 4 * * 1-5' # Mon-Fri 4AM UTC
workflow_dispatch: {}
concurrency: 'benchmarks' # Ensure only one of this runs at a time
jobs:
run-benchmarks:
name: Clickhouse queries
runs-on: ubuntu-20.04
environment: clickhouse-benchmarks
# Benchmarks are expensive to run so we only run them (periodically) against master branch and for PRs labeled `performance`
if: ${{ github.repository == 'PostHog/posthog' && (github.ref == 'refs/heads/master' || contains(github.event.pull_request.labels.*.name, 'performance')) }}
env:
DATABASE_URL: 'postgres://posthog:posthog@localhost:5432/posthog'
REDIS_URL: 'redis://localhost'
DEBUG: '1'
CLICKHOUSE_DATABASE: posthog
CLICKHOUSE_HOST: ${{ secrets.BENCHMARKS_CLICKHOUSE_HOST }}
CLICKHOUSE_USER: ${{ secrets.BENCHMARKS_CLICKHOUSE_USER }}
CLICKHOUSE_PASSWORD: ${{ secrets.BENCHMARKS_CLICKHOUSE_PASSWORD }}
CLICKHOUSE_SECURE: 'false'
CLICKHOUSE_VERIFY: 'false'
SECRET_KEY: '6b01eee4f945ca25045b5aab440b953461faf08693a9abbf1166dc7c6b9772da' # unsafe - for testing only
BENCHMARK: '1'
steps:
- uses: actions/checkout@v3
with:
# Checkout repo with full history
fetch-depth: 0
- name: Check out PostHog/benchmarks-results repo
uses: actions/checkout@v3
with:
path: ee/benchmarks/results
repository: PostHog/benchmark-results
token: ${{ secrets.POSTHOG_BOT_GITHUB_TOKEN }}
- name: Stop/Start stack with Docker Compose
run: |
docker compose -f docker-compose.dev.yml down
docker compose -f docker-compose.dev.yml up -d
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: 3.10.10
cache: 'pip'
cache-dependency-path: '**/requirements*.txt'
token: ${{ secrets.POSTHOG_BOT_GITHUB_TOKEN }}
- uses: syphar/restore-virtualenv@v1
id: cache-benchmark-tests
- name: Install SAML (python3-saml) dependencies
shell: bash
run: |
sudo apt-get update
sudo apt-get install libxml2-dev libxmlsec1-dev libxmlsec1-openssl
- name: Install python dependencies
if: steps.cache-benchmark-tests.outputs.cache-hit != 'true'
run: |
python -m pip install -r requirements-dev.txt
python -m pip install -r requirements.txt
- name: Install asv
run: python -m pip install asv==0.5.1 virtualenv
- name: Set up PostHog
run: |
python manage.py migrate & wait
python manage.py setup_dev --no-data
- name: Configure benchmarks
run: asv machine --config ee/benchmarks/asv.conf.json --yes --machine ci-benchmarks
- name: Run benchmarks
run: asv run --config ee/benchmarks/asv.conf.json --show-stderr --strict
- name: Compare results
run: |
asv compare $(cat ee/benchmarks/results/last-master-commit) HEAD --config ee/benchmarks/asv.conf.json --factor 1.2 | tee pr_vs_master.txt
asv compare $(cat ee/benchmarks/results/last-master-commit) HEAD --config ee/benchmarks/asv.conf.json --factor 1.2 --only-changed | tee pr_vs_master_changed.txt
- name: Save last benchmarked commit
if: ${{ github.ref == 'refs/heads/master' }}
run: echo "${{ github.sha }}" | tee ee/benchmarks/results/last-master-commit
- name: Generate HTML report of results
if: ${{ github.ref == 'refs/heads/master' }}
run: asv publish --config ee/benchmarks/asv.conf.json
- name: Commit update for benchmark results
if: ${{ github.repository == 'PostHog/posthog' && github.ref == 'refs/heads/master' }}
uses: stefanzweifel/git-auto-commit-action@v5
with:
repository: ee/benchmarks/results
branch: master
commit_message: 'Save benchmark results'
commit_user_name: PostHog Bot
commit_user_email: hey@posthog.com
commit_author: PostHog Bot <hey@posthog.com>
- name: Upload results as artifacts
uses: actions/upload-artifact@v3
with:
name: benchmarks
path: |
pr_vs_master.txt
pr_vs_master_changed.txt
- name: Read benchmark output
if: ${{ github.event_name == 'pull_request' }}
id: pr_vs_master_changed
uses: juliangruber/read-file-action@v1
with:
path: pr_vs_master_changed.txt
- name: Read benchmark output (full)
if: ${{ github.event_name == 'pull_request' }}
id: pr_vs_master
uses: juliangruber/read-file-action@v1
with:
path: pr_vs_master.txt
- name: Find Comment
if: ${{ github.event_name == 'pull_request' }}
uses: peter-evans/find-comment@v2
id: fc
with:
issue-number: ${{ github.event.number }}
comment-author: 'github-actions[bot]'
body-includes: ClickHouse query benchmark results from GitHub Actions
- name: Create or update comment
if: ${{ github.event_name == 'pull_request' }}
uses: peter-evans/create-or-update-comment@v3
with:
comment-id: ${{ steps.fc.outputs.comment-id }}
issue-number: ${{ github.event.number }}
body: |
ClickHouse query benchmark results from GitHub Actions
Lower numbers are good, higher numbers are bad. A ratio less than 1
means a speed up and greater than 1 means a slowdown. Green lines
beginning with `+` are slowdowns (the PR is slower then master or
master is slower than the previous release). Red lines beginning
with `-` are speedups. Blank means no changes.
Significantly changed benchmark results (PR vs master)
```diff
${{ steps.pr_vs_master_changed.outputs.content }}
```
<details>
<summary>Click to view full benchmark results</summary>
```diff
${{ steps.pr_vs_master.outputs.content }}
```
</details>
edit-mode: replace