Skip to content

Commit afcee8d

Browse files
eh-ameh-am
authored
feat: timelines (grafana/phlare#635)
Add support for timelines Also: * The resolution calculation is heavily inspired by Grafana's (https://github.com/grafana/grafana/blob/bc11a484ed97d9c87e8dd42347c2c34713e9e441/pkg/tsdb/intervalv2/intervalv2.go#L1), which is more fine grained compared to OG Pyroscope's (https://github.com/grafana/pyroscope/blob/d7ed25ebb6c50a7eab906666933d48e71573d3e8/pkg/storage/segment/constants.go#L7-L24) * Grafana's implementation uses the frontend size as a hint on how many points to calculate. Since OG Pyroscope doesn't do that (and changing it would require some effort), I am hardcoding a default, which can be revised later.
1 parent 8c2988b commit afcee8d

File tree

9 files changed

+400
-16
lines changed

9 files changed

+400
-16
lines changed

‎package.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
"@emotion/styled": "^11.10.6",
1717
"@types/color": "^3.0.2",
1818
"@types/d3-scale": "^4.0.2",
19+
"@types/flot": "^0.0.32",
1920
"@types/history": "4.7.11",
2021
"@types/jest": "^29.5.0",
2122
"@types/jquery": "^3.5.13",
@@ -51,7 +52,7 @@
5152
"@szhsin/react-menu": "3.5.2",
5253
"graphviz-react": "^1.2.5",
5354
"jquery": "^3.6.4",
54-
"pyroscope-oss": "git+https://github.com/pyroscope-io/pyroscope.git#bb66096",
55+
"pyroscope-oss": "git+https://github.com/pyroscope-io/pyroscope.git#c93a993",
5556
"react": "^18.2.0",
5657
"react-datepicker": "^4.7.0",
5758
"react-debounce-input": "^3.2.5",

‎pkg/querier/http.go‎

Lines changed: 36 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,12 +12,14 @@ import (
1212
"github.com/prometheus/prometheus/model/labels"
1313
"github.com/prometheus/prometheus/promql/parser"
1414
"github.com/pyroscope-io/pyroscope/pkg/util/attime"
15+
"golang.org/x/sync/errgroup"
1516
"google.golang.org/grpc/codes"
1617

1718
querierv1 "github.com/grafana/phlare/api/gen/proto/go/querier/v1"
1819
"github.com/grafana/phlare/api/gen/proto/go/querier/v1/querierv1connect"
1920
typesv1 "github.com/grafana/phlare/api/gen/proto/go/types/v1"
2021
phlaremodel "github.com/grafana/phlare/pkg/model"
22+
"github.com/grafana/phlare/pkg/querier/timeline"
2123
)
2224

2325
func NewHTTPHandlers(svc querierv1connect.QuerierServiceHandler) *QueryHandlers {
@@ -82,13 +84,45 @@ func (q *QueryHandlers) Render(w http.ResponseWriter, req *http.Request) {
8284
http.Error(w, err.Error(), http.StatusBadRequest)
8385
return
8486
}
85-
res, err := q.upstream.SelectMergeStacktraces(req.Context(), connect.NewRequest(selectParams))
87+
88+
var resFlame *connect.Response[querierv1.SelectMergeStacktracesResponse]
89+
g, ctx := errgroup.WithContext(req.Context())
90+
g.Go(func() error {
91+
resFlame, err = q.upstream.SelectMergeStacktraces(ctx, connect.NewRequest(selectParams))
92+
return err
93+
})
94+
95+
timelineStep := timeline.CalcPointInterval(selectParams.Start, selectParams.End)
96+
var resSeries *connect.Response[querierv1.SelectSeriesResponse]
97+
g.Go(func() error {
98+
resSeries, err = q.upstream.SelectSeries(req.Context(),
99+
connect.NewRequest(&querierv1.SelectSeriesRequest{
100+
ProfileTypeID: selectParams.ProfileTypeID,
101+
LabelSelector: selectParams.LabelSelector,
102+
Start: selectParams.Start,
103+
End: selectParams.End,
104+
Step: timelineStep,
105+
}))
106+
107+
return err
108+
})
109+
110+
err = g.Wait()
86111
if err != nil {
87112
http.Error(w, err.Error(), http.StatusInternalServerError)
88113
return
89114
}
115+
116+
seriesVal := &typesv1.Series{}
117+
if len(resSeries.Msg.Series) == 1 {
118+
seriesVal = resSeries.Msg.Series[0]
119+
}
120+
121+
fb := ExportToFlamebearer(resFlame.Msg.Flamegraph, profileType)
122+
fb.Timeline = timeline.New(seriesVal, selectParams.Start, selectParams.End, int64(timelineStep))
123+
90124
w.Header().Add("Content-Type", "application/json")
91-
if err := json.NewEncoder(w).Encode(ExportToFlamebearer(res.Msg.Flamegraph, profileType)); err != nil {
125+
if err := json.NewEncoder(w).Encode(fb); err != nil {
92126
http.Error(w, err.Error(), http.StatusInternalServerError)
93127
return
94128
}

‎pkg/querier/timeline/calculator.go‎

Lines changed: 130 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,130 @@
1+
package timeline
2+
3+
// Heavily inspired by https://github.com/grafana/grafana/blob/bc11a484ed97d9c87e8dd42347c2c34713e9e441/pkg/tsdb/intervalv2/intervalv2.go#L1
4+
5+
import (
6+
"time"
7+
)
8+
9+
var (
10+
DefaultRes int64 = 1500
11+
DefaultMinInterval = time.Second * 10
12+
)
13+
14+
// CalcPointInterval calculates the appropriate interval between each point (aka step)
15+
// Note that its main usage is with SelectSeries, therefore its
16+
// * inputs are in ms
17+
// * output is in seconds
18+
func CalcPointInterval(fromMs int64, untilMs int64) float64 {
19+
resolution := DefaultRes
20+
21+
fromNano := fromMs * 1000000
22+
untilNano := untilMs * 1000000
23+
calculatedIntervalNano := time.Duration((untilNano - fromNano) / resolution)
24+
25+
if calculatedIntervalNano < DefaultMinInterval {
26+
return DefaultMinInterval.Seconds()
27+
}
28+
29+
return roundInterval(calculatedIntervalNano).Seconds()
30+
}
31+
32+
//nolint:gocyclo
33+
func roundInterval(interval time.Duration) time.Duration {
34+
// Notice that interval may be smaller than DefaultMinInterval, and therefore some branches may never be reached
35+
// These branches are left in case the invariant changes
36+
switch {
37+
// 0.01s
38+
case interval <= 10*time.Millisecond:
39+
return time.Millisecond * 1 // 0.001s
40+
// 0.015s
41+
case interval <= 15*time.Millisecond:
42+
return time.Millisecond * 10 // 0.01s
43+
// 0.035s
44+
case interval <= 35*time.Millisecond:
45+
return time.Millisecond * 20 // 0.02s
46+
// 0.075s
47+
case interval <= 75*time.Millisecond:
48+
return time.Millisecond * 50 // 0.05s
49+
// 0.15s
50+
case interval <= 150*time.Millisecond:
51+
return time.Millisecond * 100 // 0.1s
52+
// 0.35s
53+
case interval <= 350*time.Millisecond:
54+
return time.Millisecond * 200 // 0.2s
55+
// 0.75s
56+
case interval <= 750*time.Millisecond:
57+
return time.Millisecond * 500 // 0.5s
58+
// 1.5s
59+
case interval <= 1500*time.Millisecond:
60+
return time.Millisecond * 1000 // 1s
61+
// 3.5s
62+
case interval <= 3500*time.Millisecond:
63+
return time.Millisecond * 2000 // 2s
64+
// 7.5s
65+
case interval <= 7500*time.Millisecond:
66+
return time.Millisecond * 5000 // 5s
67+
// 12.5s
68+
case interval <= 12500*time.Millisecond:
69+
return time.Millisecond * 10000 // 10s
70+
// 17.5s
71+
case interval <= 17500*time.Millisecond:
72+
return time.Millisecond * 15000 // 15s
73+
// 25s
74+
case interval <= 25000*time.Millisecond:
75+
return time.Millisecond * 20000 // 20s
76+
// 45s
77+
case interval <= 45000*time.Millisecond:
78+
return time.Millisecond * 30000 // 30s
79+
// 1.5m
80+
case interval <= 90000*time.Millisecond:
81+
return time.Millisecond * 60000 // 1m
82+
// 3.5m
83+
case interval <= 210000*time.Millisecond:
84+
return time.Millisecond * 120000 // 2m
85+
// 7.5m
86+
case interval <= 450000*time.Millisecond:
87+
return time.Millisecond * 300000 // 5m
88+
// 12.5m
89+
case interval <= 750000*time.Millisecond:
90+
return time.Millisecond * 600000 // 10m
91+
// 17.5m
92+
case interval <= 1050000*time.Millisecond:
93+
return time.Millisecond * 900000 // 15m
94+
// 25m
95+
case interval <= 1500000*time.Millisecond:
96+
return time.Millisecond * 1200000 // 20m
97+
// 45m
98+
case interval <= 2700000*time.Millisecond:
99+
return time.Millisecond * 1800000 // 30m
100+
// 1.5h
101+
case interval <= 5400000*time.Millisecond:
102+
return time.Millisecond * 3600000 // 1h
103+
// 2.5h
104+
case interval <= 9000000*time.Millisecond:
105+
return time.Millisecond * 7200000 // 2h
106+
// 4.5h
107+
case interval <= 16200000*time.Millisecond:
108+
return time.Millisecond * 10800000 // 3h
109+
// 9h
110+
case interval <= 32400000*time.Millisecond:
111+
return time.Millisecond * 21600000 // 6h
112+
// 24h
113+
case interval <= 86400000*time.Millisecond:
114+
return time.Millisecond * 43200000 // 12h
115+
// 48h
116+
case interval <= 172800000*time.Millisecond:
117+
return time.Millisecond * 86400000 // 24h
118+
// 1w
119+
case interval <= 604800000*time.Millisecond:
120+
return time.Millisecond * 86400000 // 24h
121+
// 3w
122+
case interval <= 1814400000*time.Millisecond:
123+
return time.Millisecond * 604800000 // 1w
124+
// 2y
125+
case interval < 3628800000*time.Millisecond:
126+
return time.Millisecond * 2592000000 // 30d
127+
default:
128+
return time.Millisecond * 31536000000 // 1y
129+
}
130+
}

‎pkg/querier/timeline/calculator_test.go‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
package timeline_test
2+
3+
import (
4+
"testing"
5+
"time"
6+
7+
"github.com/stretchr/testify/assert"
8+
9+
"github.com/grafana/phlare/pkg/querier/timeline"
10+
)
11+
12+
func Test_CalcPointInterval(t *testing.T) {
13+
TestDate := time.Date(2023, time.April, 18, 1, 2, 3, 4, time.UTC)
14+
15+
testCases := []struct {
16+
name string
17+
start time.Time
18+
end time.Time
19+
want int64
20+
}{
21+
{name: "1 second", start: TestDate, end: TestDate.Add(1 * time.Second), want: 10},
22+
{name: "1 hour", start: TestDate, end: TestDate.Add(1 * time.Hour), want: 10},
23+
{name: "7 days", start: TestDate, end: TestDate.Add(7 * 24 * time.Hour), want: 300},
24+
{name: "30 days", start: TestDate, end: TestDate.Add(30 * 24 * time.Hour), want: 1800},
25+
{name: "90 days", start: TestDate, end: TestDate.Add(30 * 24 * time.Hour), want: 1800},
26+
{name: "~6 months", start: TestDate, end: TestDate.Add(6 * 30 * 24 * time.Hour), want: 10800},
27+
{name: "~1 year", start: TestDate, end: TestDate.Add(12 * 30 * 24 * time.Hour), want: 21600},
28+
{name: "~2 years", start: TestDate, end: TestDate.Add(2 * 12 * 30 * 24 * time.Hour), want: 43200},
29+
{name: "~5 years", start: TestDate, end: TestDate.Add(5 * 12 * 30 * 24 * time.Hour), want: 86400},
30+
}
31+
32+
for _, tc := range testCases {
33+
t.Run(tc.name, func(t *testing.T) {
34+
got := timeline.CalcPointInterval(tc.start.UnixMilli(), tc.end.UnixMilli())
35+
36+
assert.Equal(t, float64(tc.want), got)
37+
})
38+
}
39+
40+
}

‎pkg/querier/timeline/timeline.go‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
package timeline
2+
3+
import (
4+
"github.com/pyroscope-io/pyroscope/pkg/structs/flamebearer"
5+
6+
v1 "github.com/grafana/phlare/api/gen/proto/go/types/v1"
7+
)
8+
9+
// New generates a FlamebearerTimeline,
10+
// backfilling any missing data with zeros
11+
// It assumes:
12+
// * Ordered
13+
// * startMs is earlier than the first series value
14+
// * endMs is after the last series value
15+
func New(series *v1.Series, startMs int64, endMs int64, durationDeltaSec int64) *flamebearer.FlamebearerTimelineV1 {
16+
// ms to seconds
17+
startSec := startMs / 1000
18+
points := series.GetPoints()
19+
res := make([]uint64, len(points))
20+
21+
if len(points) < 1 {
22+
return &flamebearer.FlamebearerTimelineV1{
23+
StartTime: startSec,
24+
DurationDelta: durationDeltaSec,
25+
Samples: backfill(startMs, endMs, durationDeltaSec),
26+
}
27+
}
28+
29+
i := 0
30+
prev := points[0]
31+
for _, curr := range points {
32+
backfillNum := sizeToBackfill(prev.Timestamp, curr.Timestamp, durationDeltaSec)
33+
34+
if backfillNum > 0 {
35+
// backfill + newValue
36+
bf := append(backfill(prev.Timestamp, curr.Timestamp, durationDeltaSec), uint64(curr.Value))
37+
38+
// break the slice
39+
first := res[:i]
40+
second := res[i:]
41+
42+
// add new backfilled items
43+
first = append(first, bf...)
44+
45+
// concatenate the three slices to form the new slice
46+
res = append(first, second...)
47+
prev = curr
48+
i = i + int(backfillNum)
49+
} else {
50+
res[i] = uint64(curr.Value)
51+
prev = curr
52+
i = i + 1
53+
}
54+
}
55+
56+
// Backfill with 0s for data that's not available
57+
firstAvailableData := points[0]
58+
lastAvailableData := points[len(points)-1]
59+
backFillHead := backfill(startMs, firstAvailableData.Timestamp, durationDeltaSec)
60+
backFillTail := backfill(lastAvailableData.Timestamp, endMs, durationDeltaSec)
61+
62+
res = append(backFillHead, res...)
63+
res = append(res, backFillTail...)
64+
65+
timeline := &flamebearer.FlamebearerTimelineV1{
66+
StartTime: startSec,
67+
DurationDelta: durationDeltaSec,
68+
Samples: res,
69+
}
70+
71+
return timeline
72+
}
73+
74+
// sizeToBackfill indicates how many items are needed to backfill
75+
// if none are needed, a negative value is returned
76+
func sizeToBackfill(startMs int64, endMs int64, stepSec int64) int64 {
77+
startSec := startMs / 1000
78+
endSec := endMs / 1000
79+
size := ((endSec - startSec) / stepSec) - 1
80+
return size
81+
}
82+
83+
func backfill(startMs int64, endMs int64, stepSec int64) []uint64 {
84+
size := sizeToBackfill(startMs, endMs, stepSec)
85+
if size <= 0 {
86+
size = 0
87+
}
88+
return make([]uint64, size)
89+
}

0 commit comments

Comments
 (0)