diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 000000000..f2cafdc05 --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,90 @@ +name: Luau Benchmarks + +on: + pull_request: + paths-ignore: + - "docs/**" + - "papers/**" + - "rfcs/**" + - "*.md" + - "prototyping/**" + +jobs: + benchmarks-run: + name: Run ${{ matrix.bench.title }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + engine: + - { channel: stable, version: latest, runner: "robloxdev-cli" } + # - { channel: develop, version: latest, runner: 'robloxdev-cli' } + # covers environment closest to RobloxStudio, but not available on Linux + # - { channel: develop, version: latest, runner: 'roblox-cli'} + bench: + - { + script: "run-benchmarks", + timeout: 12, + title: "Luau Benchmarks", + cachegrindTitle: "Performance", + cachegrindIterCount: 20, + } + + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v3 + + - name: Build Luau + run: make config=release luau luau-analyze + + - uses: actions/setup-python@v3 + with: + python-version: "3.9" + architecture: "x64" + + - name: Install python dependencies + run: | + python -m pip install requests + python -m pip install --user numpy scipy matplotlib ipython jupyter pandas sympy nose + + - name: Run benchmark + run: | + python bench/bench.py | tee ../${{ matrix.bench.script }}-output.txt + + - name: Navigate to root dir, create gh-pages dir then navigate into it + run: cd .. && mkdir gh-pages && cd gh-pages + + - name: Switch to & checkout gh-pages repo + uses: actions/checkout@v3 + with: + repository: AllanJeremy/luau-benchmark-results + ref: "main" + + - name: Checkout gh-pages + run: | + git status + git fetch --no-tags --depth=1 origin main + git checkout main + git pull + git status + + - name: Navigate back to luau before checking out original repo + run: cd .. + + #Run `github-action-benchmark` action + - name: Store ${{ matrix.bench.title }} result + uses: Roblox/rhysd-github-action-benchmark@v-luau + with: + name: ${{ matrix.bench.title }} + tool: "benchmarkluau" + # This should come from Luau repo + output-file-path: ../${{ matrix.bench.script }}-output.txt + + # This should map to thebenchmark results repo + external-data-json-path: ../gh-pages/dev/bench/data.json + alert-threshold: 150% + fail-threshold: 200% + fail-on-alert: true + comment-on-alert: true + comment-always: true + github-token: ${{ secrets.GITHUB_TOKEN }}