Skip to content

Commit

Permalink
Merge pull request #36 from parca-dev/storage-stripeSeries
Browse files Browse the repository at this point in the history 
pkg/storage: Create stripeSeries to improve BenchmarkHeadQuerier_Select
  • Loading branch information
@brancz
brancz committed Aug 18, 2021
2 parents a52357c + e73b5ee commit 31d7b05
Show file tree
Hide file tree
Showing 11 changed files with 472 additions and 54 deletions.
2 changes: 1 addition & 1 deletion pkg/query/query.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ func (q *Query) QueryRange(ctx context.Context, req *pb.QueryRangeRequest) (*pb.
}

start := req.Start.AsTime()
end := req.Start.AsTime()
end := req.End.AsTime()

// Timestamps don't have to match exactly and staleness kicks in within 5
// minutes of no samples, so we need to search the range of -5min to +5min
Expand Down
21 changes: 15 additions & 6 deletions pkg/query/query_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ func Test_QueryRange_EmptyStore(t *testing.T) {

// Query last 5 minutes
end := time.Now()
start := end.Add(5 * time.Minute)
start := end.Add(-5 * time.Minute)

resp, err := q.QueryRange(ctx, &pb.QueryRangeRequest{
Query: "allocs",
Expand Down Expand Up @@ -57,11 +57,15 @@ func Test_QueryRange_Valid(t *testing.T) {
p, err := profile.Parse(f)
require.NoError(t, err)

app.Append(storage.ProfileFromPprof(s, p))
// Overwrite the profile's timestamp to be within the last 5min.
p.TimeNanos = time.Now().UnixNano()

err = app.Append(storage.ProfileFromPprof(s, p))
require.NoError(t, err)

// Query last 5 minutes
end := time.Now()
start := end.Add(5 * time.Minute)
start := end.Add(-5 * time.Minute)

resp, err := q.QueryRange(ctx, &pb.QueryRangeRequest{
Query: "allocs",
Expand Down Expand Up @@ -108,12 +112,17 @@ func Test_QueryRange_Limited(t *testing.T) {
},
})
require.NoError(t, err)
app.Append(storage.ProfileFromPprof(s, p))

// Overwrite the profile's timestamp to be within the last 5min.
p.TimeNanos = time.Now().UnixNano()

err = app.Append(storage.ProfileFromPprof(s, p))
require.NoError(t, err)
}

// Query last 5 minutes
end := time.Now()
start := end.Add(5 * time.Minute)
start := end.Add(-5 * time.Minute)

limit := rand.Intn(numSeries)
resp, err := q.QueryRange(ctx, &pb.QueryRangeRequest{
Expand All @@ -133,7 +142,7 @@ func Test_QueryRange_Limited(t *testing.T) {
func Test_QueryRange_InputValidation(t *testing.T) {
ctx := context.Background()
end := time.Now()
start := end.Add(5 * time.Minute)
start := end.Add(-5 * time.Minute)

tests := map[string]struct {
req *pb.QueryRangeRequest
Expand Down
130 changes: 130 additions & 0 deletions pkg/storage/benchmark/head-querier-select.txt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,130 @@
goos: linux
goarch: amd64
pkg: github.com/parca-dev/parca/pkg/storage
cpu: AMD Ryzen 9 3900X 12-Core Processor
BenchmarkHeadQuerier_Select/1of1000000-24 10 127331008 ns/op 72236288 B/op 463 allocs/op
BenchmarkHeadQuerier_Select/10of1000000-24 10 123802009 ns/op 72236288 B/op 463 allocs/op
BenchmarkHeadQuerier_Select/100of1000000-24 10 125920239 ns/op 72236289 B/op 463 allocs/op
BenchmarkHeadQuerier_Select/1000of1000000-24 10 129210526 ns/op 72236288 B/op 463 allocs/op
BenchmarkHeadQuerier_Select/10000of1000000-24 10 124908547 ns/op 72236288 B/op 463 allocs/op
BenchmarkHeadQuerier_Select/100000of1000000-24 10 131144862 ns/op 72236288 B/op 463 allocs/op
BenchmarkHeadQuerier_Select/1000000of1000000-24 10 134257777 ns/op 72236288 B/op 463 allocs/op
PASS
ok github.com/parca-dev/parca/pkg/storage 27.277s
goos: linux
goarch: amd64
pkg: github.com/parca-dev/parca/pkg/storage
cpu: AMD Ryzen 9 3900X 12-Core Processor
BenchmarkHeadQuerier_Select/1of1000000-24 10 134532772 ns/op 72236288 B/op 463 allocs/op
BenchmarkHeadQuerier_Select/10of1000000-24 10 119641366 ns/op 72236288 B/op 463 allocs/op
BenchmarkHeadQuerier_Select/100of1000000-24 10 125914887 ns/op 72236288 B/op 463 allocs/op
BenchmarkHeadQuerier_Select/1000of1000000-24 10 129930701 ns/op 72236288 B/op 463 allocs/op
BenchmarkHeadQuerier_Select/10000of1000000-24 10 131759644 ns/op 72236288 B/op 463 allocs/op
BenchmarkHeadQuerier_Select/100000of1000000-24 10 132266590 ns/op 72236288 B/op 463 allocs/op
BenchmarkHeadQuerier_Select/1000000of1000000-24 10 137597696 ns/op 72236288 B/op 463 allocs/op
PASS
ok github.com/parca-dev/parca/pkg/storage 25.589s
goos: linux
goarch: amd64
pkg: github.com/parca-dev/parca/pkg/storage
cpu: AMD Ryzen 9 3900X 12-Core Processor
BenchmarkHeadQuerier_Select/1of1000000-24 10 127328522 ns/op 72236288 B/op 463 allocs/op
BenchmarkHeadQuerier_Select/10of1000000-24 10 125562162 ns/op 72236288 B/op 463 allocs/op
BenchmarkHeadQuerier_Select/100of1000000-24 10 125745609 ns/op 72236288 B/op 463 allocs/op
BenchmarkHeadQuerier_Select/1000of1000000-24 10 122424338 ns/op 72236288 B/op 463 allocs/op
BenchmarkHeadQuerier_Select/10000of1000000-24 10 122554449 ns/op 72236288 B/op 463 allocs/op
BenchmarkHeadQuerier_Select/100000of1000000-24 10 121190147 ns/op 72236288 B/op 463 allocs/op
BenchmarkHeadQuerier_Select/1000000of1000000-24 10 144337443 ns/op 72236288 B/op 463 allocs/op
PASS
ok github.com/parca-dev/parca/pkg/storage 24.063s
goos: linux
goarch: amd64
pkg: github.com/parca-dev/parca/pkg/storage
cpu: AMD Ryzen 9 3900X 12-Core Processor
BenchmarkHeadQuerier_Select/1of1000000-24 10 124962170 ns/op 72236288 B/op 463 allocs/op
BenchmarkHeadQuerier_Select/10of1000000-24 10 127256342 ns/op 72236288 B/op 463 allocs/op
BenchmarkHeadQuerier_Select/100of1000000-24 10 141061471 ns/op 72236288 B/op 463 allocs/op
BenchmarkHeadQuerier_Select/1000of1000000-24 10 131732303 ns/op 72236288 B/op 463 allocs/op
BenchmarkHeadQuerier_Select/10000of1000000-24 10 127100601 ns/op 72236289 B/op 463 allocs/op
BenchmarkHeadQuerier_Select/100000of1000000-24 10 123260416 ns/op 72236288 B/op 463 allocs/op
BenchmarkHeadQuerier_Select/1000000of1000000-24 10 138275214 ns/op 72236289 B/op 463 allocs/op
PASS
ok github.com/parca-dev/parca/pkg/storage 22.833s
goos: linux
goarch: amd64
pkg: github.com/parca-dev/parca/pkg/storage
cpu: AMD Ryzen 9 3900X 12-Core Processor
BenchmarkHeadQuerier_Select/1of1000000-24 10 129122031 ns/op 72236288 B/op 463 allocs/op
BenchmarkHeadQuerier_Select/10of1000000-24 10 129311444 ns/op 72236289 B/op 463 allocs/op
BenchmarkHeadQuerier_Select/100of1000000-24 10 127555004 ns/op 72236288 B/op 463 allocs/op
BenchmarkHeadQuerier_Select/1000of1000000-24 10 135287228 ns/op 72236288 B/op 463 allocs/op
BenchmarkHeadQuerier_Select/10000of1000000-24 10 127950974 ns/op 72236288 B/op 463 allocs/op
BenchmarkHeadQuerier_Select/100000of1000000-24 10 125928434 ns/op 72236288 B/op 463 allocs/op
BenchmarkHeadQuerier_Select/1000000of1000000-24 10 143721709 ns/op 72236288 B/op 463 allocs/op
PASS
ok github.com/parca-dev/parca/pkg/storage 28.617s
goos: linux
goarch: amd64
pkg: github.com/parca-dev/parca/pkg/storage
cpu: AMD Ryzen 9 3900X 12-Core Processor
BenchmarkHeadQuerier_Select/1of1000000-24 10 127525753 ns/op 72236288 B/op 463 allocs/op
BenchmarkHeadQuerier_Select/10of1000000-24 10 125712371 ns/op 72236288 B/op 463 allocs/op
BenchmarkHeadQuerier_Select/100of1000000-24 10 122850950 ns/op 72236288 B/op 463 allocs/op
BenchmarkHeadQuerier_Select/1000of1000000-24 10 128558805 ns/op 72236289 B/op 463 allocs/op
BenchmarkHeadQuerier_Select/10000of1000000-24 10 121922347 ns/op 72236288 B/op 463 allocs/op
BenchmarkHeadQuerier_Select/100000of1000000-24 10 127792038 ns/op 72236289 B/op 463 allocs/op
BenchmarkHeadQuerier_Select/1000000of1000000-24 10 141877417 ns/op 72236288 B/op 463 allocs/op
PASS
ok github.com/parca-dev/parca/pkg/storage 29.897s
goos: linux
goarch: amd64
pkg: github.com/parca-dev/parca/pkg/storage
cpu: AMD Ryzen 9 3900X 12-Core Processor
BenchmarkHeadQuerier_Select/1of1000000-24 10 124180225 ns/op 72236288 B/op 463 allocs/op
BenchmarkHeadQuerier_Select/10of1000000-24 10 122060096 ns/op 72236288 B/op 463 allocs/op
BenchmarkHeadQuerier_Select/100of1000000-24 10 124110341 ns/op 72236288 B/op 463 allocs/op
BenchmarkHeadQuerier_Select/1000of1000000-24 10 124455008 ns/op 72236288 B/op 463 allocs/op
BenchmarkHeadQuerier_Select/10000of1000000-24 10 129775053 ns/op 72236288 B/op 463 allocs/op
BenchmarkHeadQuerier_Select/100000of1000000-24 10 131115877 ns/op 72236289 B/op 463 allocs/op
BenchmarkHeadQuerier_Select/1000000of1000000-24 10 141893692 ns/op 72236288 B/op 463 allocs/op
PASS
ok github.com/parca-dev/parca/pkg/storage 21.191s
goos: linux
goarch: amd64
pkg: github.com/parca-dev/parca/pkg/storage
cpu: AMD Ryzen 9 3900X 12-Core Processor
BenchmarkHeadQuerier_Select/1of1000000-24 10 125001745 ns/op 72236288 B/op 463 allocs/op
BenchmarkHeadQuerier_Select/10of1000000-24 10 122485592 ns/op 72236288 B/op 463 allocs/op
BenchmarkHeadQuerier_Select/100of1000000-24 10 126569376 ns/op 72236288 B/op 463 allocs/op
BenchmarkHeadQuerier_Select/1000of1000000-24 10 121766572 ns/op 72236288 B/op 463 allocs/op
BenchmarkHeadQuerier_Select/10000of1000000-24 10 128793260 ns/op 72236288 B/op 463 allocs/op
BenchmarkHeadQuerier_Select/100000of1000000-24 10 126706778 ns/op 72236288 B/op 463 allocs/op
BenchmarkHeadQuerier_Select/1000000of1000000-24 10 141370904 ns/op 72236288 B/op 463 allocs/op
PASS
ok github.com/parca-dev/parca/pkg/storage 29.749s
goos: linux
goarch: amd64
pkg: github.com/parca-dev/parca/pkg/storage
cpu: AMD Ryzen 9 3900X 12-Core Processor
BenchmarkHeadQuerier_Select/1of1000000-24 10 127287199 ns/op 72236288 B/op 463 allocs/op
BenchmarkHeadQuerier_Select/10of1000000-24 10 126518865 ns/op 72236288 B/op 463 allocs/op
BenchmarkHeadQuerier_Select/100of1000000-24 10 125652103 ns/op 72236288 B/op 463 allocs/op
BenchmarkHeadQuerier_Select/1000of1000000-24 10 134784136 ns/op 72236288 B/op 463 allocs/op
BenchmarkHeadQuerier_Select/10000of1000000-24 10 122095400 ns/op 72236288 B/op 463 allocs/op
BenchmarkHeadQuerier_Select/100000of1000000-24 10 123651802 ns/op 72236288 B/op 463 allocs/op
BenchmarkHeadQuerier_Select/1000000of1000000-24 10 139163808 ns/op 72236288 B/op 463 allocs/op
PASS
ok github.com/parca-dev/parca/pkg/storage 22.747s
goos: linux
goarch: amd64
pkg: github.com/parca-dev/parca/pkg/storage
cpu: AMD Ryzen 9 3900X 12-Core Processor
BenchmarkHeadQuerier_Select/1of1000000-24 10 133705366 ns/op 72236288 B/op 463 allocs/op
BenchmarkHeadQuerier_Select/10of1000000-24 10 121820993 ns/op 72236288 B/op 463 allocs/op
BenchmarkHeadQuerier_Select/100of1000000-24 10 134766702 ns/op 72236289 B/op 463 allocs/op
BenchmarkHeadQuerier_Select/1000of1000000-24 10 131375807 ns/op 72236288 B/op 463 allocs/op
BenchmarkHeadQuerier_Select/10000of1000000-24 10 129011396 ns/op 72236288 B/op 463 allocs/op
BenchmarkHeadQuerier_Select/100000of1000000-24 10 124535479 ns/op 72236288 B/op 463 allocs/op
BenchmarkHeadQuerier_Select/1000000of1000000-24 10 137281541 ns/op 72236288 B/op 463 allocs/op
PASS
ok github.com/parca-dev/parca/pkg/storage 24.795s
8 changes: 4 additions & 4 deletions pkg/storage/chunkenc/chunk_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -121,10 +121,10 @@ func testChunk(t *testing.T, c Chunk) {
require.Equal(t, []int64{0, 0, 0, 0, 0, 0, 0, 0, 0, 42}, res4)
}

// for i in {1..10}; do go test -bench=BenchmarkIterators --benchtime=500000000x ./chunkenc >> benchmark/iterator-same.txt; done
// for i in {1..10}; do go test -bench=BenchmarkIterators --benchtime=500000000x ./chunkenc >> benchmark/iterator-increasing.txt; done
// for i in {1..10}; do go test -bench=BenchmarkIterators --benchtime=500000000x ./chunkenc >> benchmark/iterator-random.txt; done
// benchstat benchmark/iterator-same.txt benchmark/iterator-increasing.txt benchmark/iterator-random.txt >> benchmark/iterator-benchstat.txt
// for i in {1..10}; do go test -bench=BenchmarkIterators --benchtime=500000000x ./pkg/storage/chunkenc >> pkg/storage/benchmark/iterator-same.txt; done
// for i in {1..10}; do go test -bench=BenchmarkIterators --benchtime=500000000x ./pkg/storage/chunkenc >> pkg/storage/benchmark/iterator-increasing.txt; done
// for i in {1..10}; do go test -bench=BenchmarkIterators --benchtime=500000000x ./pkg/storage/chunkenc >> pkg/storage/benchmark/iterator-random.txt; done
// benchstat pkg/storage/benchmark/iterator-same.txt pkg/storage/benchmark/iterator-increasing.txt pkg/storage/benchmark/iterator-random.txt >> pkg/storage/benchmark/iterator-benchstat.txt

func BenchmarkIterators(b *testing.B) {
for _, c := range []struct {
Expand Down
23 changes: 23 additions & 0 deletions pkg/storage/db_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,3 +63,26 @@ func TestDB(t *testing.T) {
}
}
}

func TestSliceSeriesSet(t *testing.T) {
ss := SliceSeriesSet{
series: []Series{},
i: -1,
}

for i := 0; i < 100; i++ {
ss.series = append(ss.series, &MemSeries{
id: uint64(i),
minTime: 0,
maxTime: int64(i),
})
}

// Iterate over all series
for i := 0; i < 100; i++ {
require.True(t, ss.Next())
require.Equal(t, &MemSeries{id: uint64(i), maxTime: int64(i)}, ss.At())
}
require.NoError(t, ss.Err())
require.False(t, ss.Next())
}
Loading

0 comments on commit 31d7b05

Please sign in to comment.