Skip to content

Commit a93551f

Browse files
authored
Merge pull request #20181 from AwesomePatrol/test-for-contract-usage
Add Kubernetes API coverage test consuming traces
2 parents 85b7dd4 + 587388f commit a93551f

File tree

5 files changed

+228
-1
lines changed

5 files changed

+228
-1
lines changed

tests/go.mod

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ require (
2424
github.com/grpc-ecosystem/go-grpc-middleware v1.3.0
2525
github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.1.0
2626
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.1
27+
github.com/olekukonko/tablewriter v1.0.7
2728
github.com/prometheus/client_golang v1.22.0
2829
github.com/prometheus/client_model v0.6.2
2930
github.com/prometheus/common v0.65.0
@@ -80,7 +81,6 @@ require (
8081
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
8182
github.com/olekukonko/errors v0.0.0-20250405072817-4e6d85265da6 // indirect
8283
github.com/olekukonko/ll v0.0.8 // indirect
83-
github.com/olekukonko/tablewriter v1.0.7 // indirect
8484
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
8585
github.com/prometheus/procfs v0.15.1 // indirect
8686
github.com/rivo/uniseg v0.4.7 // indirect

tests/robustness/coverage/README.md

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
## Overview
2+
3+
Go tests in this directory analyze the usage of Etcd API based on collected
4+
traces from a Kubernetes cluster. They output information on:
5+
6+
1. Number of calls per gRPC method used by Kubernetes
7+
8+
This information can be used to track the coverage of k8s-etcd contract.
9+
10+
### Manual test execution
11+
12+
At first we will manually set up the cluster, run e2e tests, download traces and
13+
then execute the test.
14+
15+
1\. Set up [KIND
16+
cluster](https://kind.sigs.k8s.io/docs/user/quick-start/#installation) with
17+
tracing exporting to [Jaeger](https://www.jaegertracing.io/)
18+
19+
```
20+
export KUBECONFIG="$(pwd)/kind-with-tracing-config"
21+
kind create cluster --config kind-with-tracing.yaml
22+
kubectl run jaeger --overrides='{ "apiVersion": "v1", "spec": { "hostNetwork": true, "nodeName": "kind-control-plane", "tolerations": [{"effect": "NoExecute", "operator": "Exists"}]} }' \
23+
--labels='tier=control-plane' \
24+
--image jaegertracing/jaeger:2.6.0 \
25+
-- --set=extensions.jaeger_storage.backends.some_storage.memory.max_traces=2000000
26+
```
27+
28+
2\. Exercise Kubernetes API. For example, build and run Conformance tests from
29+
Kubernetes repository (this usually takes 30-40m or will time out after 1 hour):
30+
31+
```
32+
export KUBECONFIG="$(pwd)/kind-with-tracing-config"
33+
kind export kubeconfig
34+
make WHAT="test/e2e/e2e.test"
35+
./_output/bin/e2e.test -context kind-kind -ginkgo.focus=".*Conformance" -num-nodes 2
36+
```
37+
38+
3\. Download traces and put them into `tests/robustness/coverage/testdata`
39+
directory in Etcd git repository:
40+
41+
```
42+
kubectl port-forward jaeger --address localhost --address :: 16686:16686 &
43+
curl -v --get --retry 10 --retry-connrefused -o testdata/demo_traces.json \
44+
-H "Content-Type: application/json" \
45+
--data-urlencode "query.start_time_min=$(date --date="5 days ago" -Ins)" \
46+
--data-urlencode "query.start_time_max=$(date -Ins)" \
47+
--data-urlencode "query.service_name=etcd" \
48+
"http://127.0.0.1:16686/api/v3/traces"
49+
kill $!
50+
```
51+
52+
4\. Run Go test
53+
54+
```
55+
go test -v -timeout 60s go.etcd.io/etcd/tests/v3/robustness/coverage
56+
```
57+
58+
### Automated test execution
59+
60+
Work on improving these tests is tracked in https://github.com/etcd-io/etcd/issues/20182
Lines changed: 149 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,149 @@
1+
// Copyright 2025 The etcd Authors
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package coverage_test
16+
17+
import (
18+
"bytes"
19+
"encoding/json"
20+
"fmt"
21+
"os"
22+
"path/filepath"
23+
"strings"
24+
"testing"
25+
26+
"github.com/olekukonko/tablewriter"
27+
"github.com/olekukonko/tablewriter/tw"
28+
traceservice "go.opentelemetry.io/proto/otlp/collector/trace/v1"
29+
tracev1 "go.opentelemetry.io/proto/otlp/trace/v1"
30+
"google.golang.org/protobuf/encoding/protojson"
31+
)
32+
33+
func TestInterfaceUse(t *testing.T) {
34+
files, err := os.ReadDir("testdata")
35+
if err != nil {
36+
t.Fatal(err)
37+
}
38+
39+
for _, file := range files {
40+
filename := file.Name()
41+
if filename == ".gitignore" {
42+
continue
43+
}
44+
t.Run(filename, func(t *testing.T) { testInterfaceUse(t, filename) })
45+
}
46+
}
47+
48+
func testInterfaceUse(t *testing.T, filename string) {
49+
b, err := os.ReadFile(filepath.Join("testdata", filename))
50+
if err != nil {
51+
t.Fatalf("read test data: %v", err)
52+
}
53+
var dump Dump
54+
err = json.Unmarshal(b, &dump)
55+
if err != nil {
56+
t.Fatalf("unmarshal testdata %s: %v", filename, err)
57+
}
58+
if dump.Result == nil {
59+
t.Fatalf("missing result data")
60+
}
61+
traces := dump.Result
62+
63+
callsByOperationName := make(map[string]int)
64+
for _, trace := range traces.GetResourceSpans() {
65+
serviceName := getServiceName(trace)
66+
if serviceName != "etcd" {
67+
continue
68+
}
69+
opName := getOperationName(trace)
70+
callsByOperationName[opName]++
71+
}
72+
t.Logf("\n%s", printableCallTable(callsByOperationName))
73+
74+
knownMethodsUsedByKubernetes := map[string]bool{
75+
"etcdserverpb.KV/Range": true, // All calls should go through etcd-k8s interface
76+
"etcdserverpb.KV/Txn": true, // All calls should go through etcd-k8s interface
77+
"etcdserverpb.KV/Compact": true, // Compaction should move to using internal Etcd mechanism
78+
"etcdserverpb.Watch/Watch": true, // Not part of the contract interface (yet)
79+
"etcdserverpb.Lease/LeaseGrant": true, // Used to manage masterleases and events
80+
"etcdserverpb.Maintenance/Status": true, // Used to expose database size on apiserver's metrics endpoint
81+
}
82+
for method := range knownMethodsUsedByKubernetes {
83+
t.Run(method, func(t *testing.T) {
84+
if _, ok := callsByOperationName[method]; !ok {
85+
t.Errorf("expected %q method to be called at least once", method)
86+
}
87+
})
88+
}
89+
t.Run("only_expected_methods_were_called", func(t *testing.T) {
90+
for opName := range callsByOperationName {
91+
if !knownMethodsUsedByKubernetes[opName] {
92+
t.Errorf("method called outside the list: %s", opName)
93+
}
94+
}
95+
})
96+
}
97+
98+
type Traces struct {
99+
traceservice.ExportTraceServiceRequest
100+
}
101+
102+
func (t *Traces) UnmarshalJSON(b []byte) error {
103+
return protojson.Unmarshal(b, &t.ExportTraceServiceRequest)
104+
}
105+
106+
type Dump struct {
107+
Result *Traces `json:"result"`
108+
}
109+
110+
func getServiceName(trace *tracev1.ResourceSpans) string {
111+
for _, kv := range trace.GetResource().GetAttributes() {
112+
if kv.GetKey() == "service.name" {
113+
return kv.GetValue().GetStringValue()
114+
}
115+
}
116+
return ""
117+
}
118+
119+
func getOperationName(trace *tracev1.ResourceSpans) string {
120+
for _, scopeSpan := range trace.GetScopeSpans() {
121+
for _, span := range scopeSpan.GetSpans() {
122+
name := span.GetName()
123+
if strings.HasPrefix(name, "etcdserverpb") {
124+
return name
125+
}
126+
}
127+
}
128+
return ""
129+
}
130+
131+
func printableCallTable(callsByOperationName map[string]int) string {
132+
buf := new(bytes.Buffer)
133+
cfgBuilder := tablewriter.NewConfigBuilder().WithRowAlignment(tw.AlignRight)
134+
table := tablewriter.NewTable(buf, tablewriter.WithConfig(cfgBuilder.Build()))
135+
table.Header("method", "calls", "percent")
136+
137+
totalCalls := 0
138+
for _, c := range callsByOperationName {
139+
totalCalls += c
140+
}
141+
142+
for opName, callCount := range callsByOperationName {
143+
table.Append(opName, callCount, fmt.Sprintf("%.2f%%", float64(callCount*100)/float64(totalCalls)))
144+
}
145+
table.Footer("total", totalCalls, "100.00%")
146+
147+
table.Render()
148+
return buf.String()
149+
}
Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
kind: Cluster
2+
apiVersion: kind.x-k8s.io/v1alpha4
3+
nodes:
4+
- role: control-plane
5+
kubeadmConfigPatches:
6+
- |
7+
kind: ClusterConfiguration
8+
etcd:
9+
local:
10+
extraArgs:
11+
experimental-enable-distributed-tracing: "true"
12+
experimental-distributed-tracing-address: "0.0.0.0:4317"
13+
experimental-distributed-tracing-service-name: "etcd"
14+
experimental-distributed-tracing-sampling-rate: "1000000"
15+
- role: worker
16+
- role: worker
Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
*
2+
!.gitignore

0 commit comments

Comments
 (0)