diff --git a/Makefile b/Makefile index 9d6bce1b..b94d867c 100644 --- a/Makefile +++ b/Makefile @@ -98,19 +98,30 @@ benchmarks: .PHONY: bench-old bench-old: benchmarks @echo "Benchmarking old engine" - @go test ./... -bench 'BenchmarkRangeQuery/.*/old_engine' -run none -count 5 | sed -u 's/\/old_engine//' > benchmarks/old.out + @go test ./... -bench 'BenchmarkRangeQuery/slow/.*/old_engine' -run none -count 5 | sed -u 's/\/old_engine//' > benchmarks/old.out @go test ./... -bench 'BenchmarkNativeHistograms/.*/old_engine' -run none -count 5 | sed -u 's/\/old_engine//' >> benchmarks/old.out .PHONY: bench-new bench-new: benchmarks @echo "Benchmarking new engine" - @go test ./... -bench 'BenchmarkRangeQuery/.*/new_engine' -run none -count 5 | sed -u 's/\/new_engine//' > benchmarks/new.out + @go test ./... -bench 'BenchmarkRangeQuery/slow/.*/new_engine' -run none -count 5 | sed -u 's/\/new_engine//' > benchmarks/new.out @go test ./... -bench 'BenchmarkNativeHistograms/.*/new_engine' -run none -count 5 | sed -u 's/\/new_engine//' >> benchmarks/new.out .PHONY: benchmark benchmark: bench-old bench-new @benchstat benchmarks/old.out benchmarks/new.out +.PHONY : bench-fast +bench-fast : benchmarks + @mkdir -p benchmarks + @echo "Benchmarking old engine" + @go test ./... -bench 'BenchmarkRangeQuery/fast/.*/old_engine' -run none -count 5 | sed -u 's/\/old_engine//' > benchmarks/old.out + @go test ./... -bench 'BenchmarkNativeHistograms/.*/old_engine' -run none -count 5 | sed -u 's/\/old_engine//' >> benchmarks/old.out + @echo "Benchmarking new engine" + @go test ./... -bench 'BenchmarkRangeQuery/fast/.*/new_engine' -run none -count 5 | sed -u 's/\/new_engine//' > benchmarks/new.out + @go test ./... -bench 'BenchmarkNativeHistograms/.*/new_engine' -run none -count 5 | sed -u 's/\/new_engine//' >> benchmarks/new.out + @benchstat benchmarks/old.out benchmarks/new.out + .PHONY: sync-parser sync-parser: @echo "Cleaning existing directories" diff --git a/engine/bench_test.go b/engine/bench_test.go index f7f6b8e9..12f275b9 100644 --- a/engine/bench_test.go +++ b/engine/bench_test.go @@ -115,15 +115,11 @@ func BenchmarkSingleQuery(b *testing.B) { func BenchmarkRangeQuery(b *testing.B) { samplesPerHour := 60 * 2 - sixHourDataset := setupStorage(b, 1000, 3, 6*samplesPerHour) - defer sixHourDataset.Close() - - largeSixHourDataset := setupStorage(b, 10000, 10, 6*samplesPerHour) - defer largeSixHourDataset.Close() - - sevenDaysAndTwoHoursDataset := setupStorage(b, 1000, 3, (7*24+2)*samplesPerHour) - defer sevenDaysAndTwoHoursDataset.Close() - + var ( + sixHourDataset *teststorage.TestStorage + largeSixHourDataset *teststorage.TestStorage + sevenDaysAndTwoHoursDataset *teststorage.TestStorage + ) start := time.Unix(0, 0) end := start.Add(2 * time.Hour) step := time.Second * 30 @@ -296,35 +292,81 @@ func BenchmarkRangeQuery(b *testing.B) { }, SelectorBatchSize: 256, } + b.Run("fast", func(b *testing.B) { + sixHourDataset := setupStorage(b, 1000, 3, 6*samplesPerHour) + defer sixHourDataset.Close() + + for _, tc := range cases { + b.Run(tc.name, func(b *testing.B) { + if tc.storage != sixHourDataset { + b.Skip() + } + b.ReportAllocs() + b.Run("old_engine", func(b *testing.B) { - for _, tc := range cases { - b.Run(tc.name, func(b *testing.B) { - b.ReportAllocs() - b.Run("old_engine", func(b *testing.B) { + promEngine := promql.NewEngine(opts.EngineOpts) - promEngine := promql.NewEngine(opts.EngineOpts) + b.ResetTimer() + b.ReportAllocs() + for i := 0; i < b.N; i++ { + qry, err := promEngine.NewRangeQuery(context.Background(), tc.storage, nil, tc.query, start, end, step) + testutil.Ok(b, err) - b.ResetTimer() - b.ReportAllocs() - for i := 0; i < b.N; i++ { - qry, err := promEngine.NewRangeQuery(context.Background(), tc.storage, nil, tc.query, start, end, step) - testutil.Ok(b, err) + oldResult := qry.Exec(context.Background()) + testutil.Ok(b, oldResult.Err) + } + }) + b.Run("new_engine", func(b *testing.B) { + b.ResetTimer() + b.ReportAllocs() - oldResult := qry.Exec(context.Background()) - testutil.Ok(b, oldResult.Err) - } + for i := 0; i < b.N; i++ { + newResult := executeRangeQuery(b, tc.query, tc.storage, start, end, step, opts) + testutil.Ok(b, newResult.Err) + } + }) }) - b.Run("new_engine", func(b *testing.B) { - b.ResetTimer() + } + }) + b.Run("slow", func(b *testing.B) { + sixHourDataset := setupStorage(b, 1000, 3, 6*samplesPerHour) + defer sixHourDataset.Close() + + largeSixHourDataset := setupStorage(b, 10000, 10, 6*samplesPerHour) + defer largeSixHourDataset.Close() + + sevenDaysAndTwoHoursDataset := setupStorage(b, 1000, 3, (7*24+2)*samplesPerHour) + defer sevenDaysAndTwoHoursDataset.Close() + + for _, tc := range cases { + b.Run(tc.name, func(b *testing.B) { b.ReportAllocs() + b.Run("old_engine", func(b *testing.B) { - for i := 0; i < b.N; i++ { - newResult := executeRangeQuery(b, tc.query, tc.storage, start, end, step, opts) - testutil.Ok(b, newResult.Err) - } + promEngine := promql.NewEngine(opts.EngineOpts) + + b.ResetTimer() + b.ReportAllocs() + for i := 0; i < b.N; i++ { + qry, err := promEngine.NewRangeQuery(context.Background(), tc.storage, nil, tc.query, start, end, step) + testutil.Ok(b, err) + + oldResult := qry.Exec(context.Background()) + testutil.Ok(b, oldResult.Err) + } + }) + b.Run("new_engine", func(b *testing.B) { + b.ResetTimer() + b.ReportAllocs() + + for i := 0; i < b.N; i++ { + newResult := executeRangeQuery(b, tc.query, tc.storage, start, end, step, opts) + testutil.Ok(b, newResult.Err) + } + }) }) - }) - } + } + }) } func BenchmarkNativeHistograms(b *testing.B) {