perf: benchmarking code improvements (#630)

* adds benchmarking code

* wip

* pull benchmarks

* adds more info to readme

* benchmark improvements

* no extra logging

* fix

* improvements

* fix

* changes

* lint fix

* initial version of pr-pull report

* fixes

* fix

* fix

* fix

* fix

* fix

* report improvements

* fix

* fix

* wip: unifying benchmark scripts

* more benchmark improvements

* updates to benchmark

* cleanup

* fixes

* fix

.dockerignore

@@ -10,3 +10,4 @@ benchmark/runs
 benchmark/docker-compose.yml
 benchmark/grafana-*
 benchmark/node_modules
+benchmark/*.env

.gitignore

@@ -18,12 +18,8 @@ pkged.go
 .idea
 .vscode
 vendor/
-benchmark/git-info.env
 benchmark/.env
-benchmark/ci/pr/dashboard-screenshots
-benchmark/ci/pr/table-report
-benchmark/ci/pr/image-report
-benchmark/ci/pr/meta-report
+benchmark/report
 /.env
 # diff output is stored per test
 cypress/snapshots/**/__diff_output__

benchmark/Dockerfile

@@ -1,3 +1,13 @@
+#                        _                     _
+#                       | |                   | |
+#  _ __  _   _ _ __ ___ | |__   ___ _ __   ___| |__
+# | '_ \| | | | '__/ _ \| '_ \ / _ \ '_ \ / __| '_ \
+# | |_) | |_| | | | (_) | |_) |  __/ | | | (__| | | |
+# | .__/ \__, |_|  \___/|_.__/ \___|_| |_|\___|_| |_|
+# | |     __/ |
+# |_|    |___/
+#
+
 FROM golang:1.16.3 as builder
 
 WORKDIR /go/src/app
@@ -19,6 +29,10 @@ FROM ubuntu:latest
 WORKDIR /var/lib/pyrobench
 
 RUN apt-get update && apt-get install ca-certificates -y && update-ca-certificates
+RUN apt-get update && apt-get install -y curl
+
+RUN curl https://pyroscope-public.s3.amazonaws.com/benchmark/fixtures.tgz | tar -xzv
+
 # Create a group and user
 #RUN addgroup -S pyrobench && adduser -S pyrobench -G pyrobench
 RUN useradd -ms /bin/bash pyrobench

benchmark/README.md

@@ -1,19 +1,39 @@
+## What is this?
+
+This is a benchmark suite. It is used to benchmark pyroscope server against various workloads, particularly data ingestion from clients in push mode and data retrieval from clients in pull mode.
+
+You can use it locally to test hypothetical scenarios, or you can use it to compare the performance of different versions of pyroscope.
+
+It uses a docker-compose file to run the whole benchmark suite. It uses multiple `docker-compose.yml` files (`*.dev.yml`, `*.ci.yml`) to augment the suite for development or ci environments.
+
+Here's an overview diagram of the whole system:
+[![architecture](./architecture.svg)](./architecture.svg)
+
 ## Prerequisites
-* docker-compose
+* You'll need `docker-compose` installed.
+* Ideally you want to be running this on a large enough machine (> 4 cores, 16 GB of RAM), otherwise services might run out of memory. See Troubleshooting guide below for more info.
 
 ## Usage
 
-To start benchmark run:
+To start benchmark locally run:
 ```
 ./start.sh
 ```
 
+## Configuration
+
+All of configuration is done with `config.env` file.
+
+To configure the benchmarking parameters, edit `config.env` file. It contains variables that are used by pyroscope and pyrobench.
 
-Pass `--wait` to make the system continue running after benchmarking is over:
+For example, the following config can be used to only test pull mode:
 ```
-sh start.sh --wait
+PYROBENCH_APPS=0
+PYROBENCH_NO_EXIT_WHEN_DONE=true
 ```
 
+You can find documentation for all of the possible flags [here](https://github.com/pyroscope-io/pyroscope/blob/3fab8a8949b3538e98af051598edf514b14eaafe/benchmark/internal/config/config.go#L14-L39)
+
 ## Browsing results
 To view results open http://localhost:8080/d/tsWRL6ReZQkirFirmyvnWX1akHXJeHT8I8emjGJo/main?orgId=1.
 
@@ -33,3 +53,28 @@ For that, just pick a big enough value for `PYROBENCH_REQUESTS`, like `100000` (
 Make sure you have enough memory allocated for docker, e.g on a mac:
 
 ![image](https://user-images.githubusercontent.com/662636/128406795-f4a50e4b-03d7-4eed-a637-45f0c638a16b.png)
+
+
+## Design Goals
+
+This benchmark suite attempts to be as flexible as possible, meaning you can use it to test different scenarios with minimum changes to the benchmark suite. We attempt to achieve this by keeping the codebase as small as possible and thus easily editable.
+
+## PR Benchmark
+
+This benchmark is also used to benchmark each pull request against the `main` branch.
+
+* it runs 2 instances of pyroscope (the one in the PR and the main one) in docker-compose.
+* generates test load against both instances
+* takes a screenshot of the dashboard panes
+* posts a PR comment using [dangerjs](https://danger.systems/js/)
+
+## Enhancing PR comment content
+
+* To add more panes edit `monitoring/benchmark-pr.jsonnet` file
+* To add more things to the report update the `queries.yml` file
+
+## TODO
+
+Future improvements:
+* [ ] better options for mock data
+

benchmark/cmd/command/banner.go

@@ -4,16 +4,16 @@ import (
 	"github.com/pyroscope-io/pyroscope/pkg/cli"
 )
 
-// made here http://patorjk.com/software/taag/#p=display&f=Doom&t=Pyrobench
+// made here http://patorjk.com/software/taag/#p=display&f=Doom&t=pyrobench
 var banner = `
-______                _                     _
-| ___ \              | |                   | |
-| |_/ /   _ _ __ ___ | |__   ___ _ __   ___| |__
-|  __/ | | | '__/ _ \| '_ \ / _ \ '_ \ / __| '_ \
-| |  | |_| | | | (_) | |_) |  __/ | | | (__| | | |
-\_|   \__, |_|  \___/|_.__/ \___|_| |_|\___|_| |_|
-       __/ |
-      |___/
+                       _                     _
+                      | |                   | |
+ _ __  _   _ _ __ ___ | |__   ___ _ __   ___| |__
+| '_ \| | | | '__/ _ \| '_ \ / _ \ '_ \ / __| '_ \
+| |_) | |_| | | | (_) | |_) |  __/ | | | (__| | | |
+| .__/ \__, |_|  \___/|_.__/ \___|_| |_|\___|_| |_|
+| |     __/ |
+|_|    |___/
 `
 
 func init() {

benchmark/cmd/command/ci-report.go

@@ -72,7 +72,7 @@ func newReport(cfg *config.Report) *cobra.Command {
 				return err
 			}
 
-			report, err := mr.Report(cfg.MetaReport.Params)
+			report, err := mr.Report(cfg.MetaReport.Title, cfg.MetaReport.Params)
 			if err != nil {
 				return err
 			}