-
Notifications
You must be signed in to change notification settings - Fork 432
/
stats.go
156 lines (134 loc) · 6.08 KB
/
stats.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
package assertions
import (
"fmt"
"net/http"
"regexp"
"time"
stats2 "go.opencensus.io/stats"
"go.opencensus.io/stats/view"
"k8s.io/utils/pointer"
"github.com/solo-io/gloo/projects/gloo/pkg/defaults"
"github.com/solo-io/gloo/test/gomega/matchers"
"github.com/solo-io/gloo/test/gomega/transforms"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/onsi/gomega/types"
"github.com/solo-io/gloo/pkg/cliutil"
"github.com/solo-io/go-utils/stats"
)
// Gloo Edge exports statistics to provide details about how the system is behaving
// Most stats utilities are defined in: https://github.com/solo-io/go-utils/tree/main/stats
// This file contains a set of assertions that can be performed by tests to ensure that recorded stats
// match what we would expect
const (
// timeToSyncStats represents the interval at which metrics reporting occurs
timeToSyncStats = time.Second * 5
// SafeTimeToSyncStats represents a safe estimate for metrics reporting interval. We provide a buffer beyond
// the time that metrics reporting occurs, to account for latencies. Tests should use the SafeTimeToSyncStats
// to ensure that we are polling the stats endpoints infrequently enough that they will have updated each time
SafeTimeToSyncStats = timeToSyncStats + time.Second*3
)
// StatsPortFwd represents the set of configuration required to generated port-forward to access statistics locally
type StatsPortFwd struct {
ResourceName string
ResourceNamespace string
LocalPort int
TargetPort int
}
// DefaultStatsPortFwd is a commonly used port-forward configuration, since Gloo Deployment stats are the most valuable
// This is used in Gloo Enterprise
var DefaultStatsPortFwd = StatsPortFwd{
ResourceName: "deployment/gloo",
ResourceNamespace: defaults.GlooSystem,
LocalPort: stats.DefaultPort,
TargetPort: stats.DefaultPort,
}
// EventuallyStatisticsMatchAssertions first opens a fort-forward and then performs
// a series of Asynchronous assertions. The fort-forward is cleaned up with the function returns
func EventuallyStatisticsMatchAssertions(statsPortFwd StatsPortFwd, assertions ...types.AsyncAssertion) {
EventuallyWithOffsetStatisticsMatchAssertions(1, statsPortFwd, assertions...)
}
// EventuallyWithOffsetStatisticsMatchAssertions first opens a fort-forward and then performs
// a series of Asynchronous assertions. The fort-forward is cleaned up with the function returns
func EventuallyWithOffsetStatisticsMatchAssertions(offset int, statsPortFwd StatsPortFwd, assertions ...types.AsyncAssertion) {
portForward, err := cliutil.PortForward(
statsPortFwd.ResourceNamespace,
statsPortFwd.ResourceName,
fmt.Sprintf("%d", statsPortFwd.LocalPort),
fmt.Sprintf("%d", statsPortFwd.TargetPort),
false)
ExpectWithOffset(offset+1, err).NotTo(HaveOccurred())
defer func() {
if portForward.Process != nil {
_ = portForward.Process.Kill()
_ = portForward.Process.Release()
}
}()
By("Ensure port-forward is open before performing assertions")
statsRequest, err := http.NewRequest(http.MethodGet, fmt.Sprintf("http://localhost:%d/", statsPortFwd.LocalPort), nil)
ExpectWithOffset(offset+1, err).NotTo(HaveOccurred())
EventuallyWithOffset(offset+1, func(g Gomega) {
resp, err := http.DefaultClient.Do(statsRequest)
g.Expect(err).NotTo(HaveOccurred())
defer resp.Body.Close()
g.Expect(resp).To(matchers.HaveHttpResponse(&matchers.HttpResponse{
StatusCode: http.StatusOK,
Body: Not(BeEmpty()),
}))
}).Should(Succeed())
By("Perform the assertions while the port forward is open")
for _, assertion := range assertions {
assertion.WithOffset(offset + 1).ShouldNot(HaveOccurred())
}
}
// IntStatisticReachesConsistentValueAssertion returns an assertion that a prometheus stats has reached a consistent value
// It optionally returns the value of that statistic as well
// Arguments:
//
// prometheusStat (string) - The name of the statistic we will be evaluating
// inARow (int) - We periodically poll the statistic value from a metrics endpoint. InARow represents
// the number of consecutive times the statistic must be the same for it to be considered "consistent"
// For example, if InARow=4, we must poll the endpoint 4 times consecutively and return the same value
func IntStatisticReachesConsistentValueAssertion(prometheusStat string, inARow int) (types.AsyncAssertion, *int) {
statRegex, err := regexp.Compile(fmt.Sprintf("%s ([\\d]+)", prometheusStat))
Expect(err).NotTo(HaveOccurred())
statTransform := transforms.IntRegexTransform(statRegex)
// Assumes that the metrics are exposed via the default port
metricsRequest, err := http.NewRequest(http.MethodPost, fmt.Sprintf("http://localhost:%d/metrics", stats.DefaultPort), nil)
Expect(err).NotTo(HaveOccurred())
var (
currentlyInARow = 0
previousStatValue = 0
currentStatValue = pointer.Int(0)
)
return Eventually(func(g Gomega) {
resp, err := http.DefaultClient.Do(metricsRequest)
g.Expect(err).NotTo(HaveOccurred())
defer resp.Body.Close()
g.Expect(resp).To(matchers.HaveHttpResponse(&matchers.HttpResponse{
StatusCode: http.StatusOK,
Body: WithTransform(func(body []byte) error {
statValue, transformErr := statTransform(body)
*currentStatValue = statValue
return transformErr
}, Not(HaveOccurred())),
}))
if *currentStatValue == 0 || *currentStatValue != previousStatValue {
currentlyInARow = 0
} else {
currentlyInARow += 1
}
previousStatValue = *currentStatValue
g.Expect(currentlyInARow).To(Equal(inARow))
}, "2m", SafeTimeToSyncStats), currentStatValue
}
func ExpectStatLastValueMatches(measure *stats2.Int64Measure, lastValueMatcher types.GomegaMatcher) {
rows, err := view.RetrieveData(measure.Name())
ExpectWithOffset(1, err).NotTo(HaveOccurred())
ExpectWithOffset(1, rows).To(WithTransform(transforms.WithLastValueTransform(), lastValueMatcher))
}
func ExpectStatSumMatches(measure *stats2.Int64Measure, sumValueMatcher types.GomegaMatcher) {
rows, err := view.RetrieveData(measure.Name())
ExpectWithOffset(1, err).NotTo(HaveOccurred())
ExpectWithOffset(1, rows).To(WithTransform(transforms.WithSumValueTransform(), sumValueMatcher))
}