Skip to content

Commit 2f67dbc

Browse files
authored
Add golden tests. (#362)
It's easy to make changes to the code generation logic that generate sdk code that appears to be correct and passes unit tests, but doesn't interact correctly with nexus. This patch introduces the concept of golden tests for nexus endpoints: we fetch real responses from a few endpoints of interest, check in the resulting json, and assert that we can unmarshal it into our generated types and marshal it back to equivalent json correctly. We also include a script and make target for refreshing golden files.
1 parent 199901c commit 2f67dbc

File tree

6 files changed

+302
-0
lines changed

6 files changed

+302
-0
lines changed

Makefile

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,11 @@ test: ## Runs the go tests.
5454
@ echo "+ Running Go tests..."
5555
@ $(GO) test -v -tags "$(BUILDTAGS)" ./...
5656

57+
.PHONY: golden-fixtures
58+
golden-fixtures: ## Refreshes golden test fixtures. Requires OXIDE_HOST, OXIDE_TOKEN, and OXIDE_PROJECT.
59+
@ echo "+ Refreshing golden test fixtures..."
60+
@ $(GO) run ./oxide/testdata/main.go
61+
5762
.PHONY: vet
5863
vet: ## Verifies `go vet` passes.
5964
@ echo "+ Verifying go vet passes..."

oxide/golden_test.go

Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,115 @@
1+
// This Source Code Form is subject to the terms of the Mozilla Public
2+
// License, v. 2.0. If a copy of the MPL was not distributed with this
3+
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
4+
5+
package oxide
6+
7+
import (
8+
"encoding/json"
9+
"os"
10+
"testing"
11+
"time"
12+
13+
"github.com/google/go-cmp/cmp"
14+
"github.com/stretchr/testify/require"
15+
)
16+
17+
// TestGoldenRoundTrip tests that real API responses can be unmarshaled and
18+
// marshaled back to equivalent JSON. This catches mismatches between our
19+
// generated types and the actual API format.
20+
//
21+
// To refresh the fixtures, run:
22+
//
23+
// go run ./oxide/testdata/main.go
24+
func TestGoldenRoundTrip(t *testing.T) {
25+
tests := []struct {
26+
name string
27+
fixture string
28+
test func(t *testing.T, fixture string)
29+
}{
30+
{
31+
name: "timeseries_query_response",
32+
fixture: "testdata/recordings/timeseries_query_response.json",
33+
test: testRoundTrip[OxqlQueryResult],
34+
},
35+
{
36+
name: "disk_list_response",
37+
fixture: "testdata/recordings/disk_list_response.json",
38+
test: testRoundTrip[DiskResultsPage],
39+
},
40+
{
41+
name: "loopback_addresses_response",
42+
fixture: "testdata/recordings/loopback_addresses_response.json",
43+
test: testRoundTrip[LoopbackAddressResultsPage],
44+
},
45+
}
46+
47+
for _, tt := range tests {
48+
t.Run(tt.name, func(t *testing.T) {
49+
tt.test(t, tt.fixture)
50+
})
51+
}
52+
}
53+
54+
func testRoundTrip[T any](t *testing.T, fixturePath string) {
55+
data, err := os.ReadFile(fixturePath)
56+
require.NoError(t, err, "failed to read fixture")
57+
58+
var typed T
59+
err = json.Unmarshal(data, &typed)
60+
require.NoError(t, err, "failed to unmarshal fixture")
61+
62+
remarshaled, err := json.Marshal(typed)
63+
require.NoError(t, err, "failed to marshal")
64+
65+
var expected, actual any
66+
require.NoError(t, json.Unmarshal(data, &expected))
67+
require.NoError(t, json.Unmarshal(remarshaled, &actual))
68+
69+
expected = stripNulls(expected)
70+
actual = stripNulls(actual)
71+
72+
if diff := cmp.Diff(expected, actual, timestampComparer()); diff != "" {
73+
t.Errorf("round-trip mismatch (-fixture +remarshaled):\n%s", diff)
74+
}
75+
}
76+
77+
// timestampComparer returns a cmp.Option that compares timestamp strings
78+
// by their actual time value, handling precision differences. Rust and go format timestamps
79+
// slightly differently, so we need to normalize to avoid spurious differences in marshalled values.
80+
func timestampComparer() cmp.Option {
81+
return cmp.Comparer(func(a, b string) bool {
82+
ta, errA := time.Parse(time.RFC3339Nano, a)
83+
tb, errB := time.Parse(time.RFC3339Nano, b)
84+
if errA == nil && errB == nil {
85+
return ta.Equal(tb)
86+
}
87+
return a == b
88+
})
89+
}
90+
91+
// stripNulls recursively removes null values from JSON-unmarshaled data. We use this workaround
92+
// because the SDK and API don't always handle null fields consistently.
93+
//
94+
// TODO: Investigate options to harmonize null handling across services so that we don't have to
95+
// pre-process the responses here.
96+
func stripNulls(v any) any {
97+
switch val := v.(type) {
98+
case map[string]any:
99+
result := make(map[string]any)
100+
for k, v := range val {
101+
if v != nil {
102+
result[k] = stripNulls(v)
103+
}
104+
}
105+
return result
106+
case []any:
107+
result := make([]any, len(val))
108+
for i, v := range val {
109+
result[i] = stripNulls(v)
110+
}
111+
return result
112+
default:
113+
return v
114+
}
115+
}

oxide/testdata/main.go

Lines changed: 179 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,179 @@
1+
// This Source Code Form is subject to the terms of the Mozilla Public
2+
// License, v. 2.0. If a copy of the MPL was not distributed with this
3+
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
4+
5+
//go:build ignore
6+
7+
// This script records real API responses for use in golden file tests.
8+
// Run with: go run ./oxide/testdata/main.go [-api-version VERSION]
9+
//
10+
// Requires OXIDE_HOST, OXIDE_TOKEN, and OXIDE_PROJECT environment variables.
11+
// Optionally pass -api-version to set the API-Version header on requests.
12+
package main
13+
14+
import (
15+
"bytes"
16+
"crypto/tls"
17+
"encoding/json"
18+
"flag"
19+
"fmt"
20+
"io"
21+
"log"
22+
"net/http"
23+
"os"
24+
"path/filepath"
25+
)
26+
27+
var (
28+
apiVersion = flag.String("api-version", "", "API version to send in requests (optional)")
29+
30+
client = &http.Client{
31+
Transport: &http.Transport{
32+
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
33+
},
34+
}
35+
)
36+
37+
func main() {
38+
flag.Parse()
39+
40+
host := os.Getenv("OXIDE_HOST")
41+
token := os.Getenv("OXIDE_TOKEN")
42+
project := os.Getenv("OXIDE_PROJECT")
43+
44+
if host == "" || token == "" || project == "" {
45+
log.Fatalf("OXIDE_HOST, OXIDE_TOKEN, and OXIDE_PROJECT environment variables must be set")
46+
}
47+
48+
if *apiVersion != "" {
49+
fmt.Printf("Using API-Version: %s\n", *apiVersion)
50+
}
51+
52+
testdataDir := "./oxide/testdata/recordings"
53+
54+
recordTimeseriesQuery(host, token, testdataDir)
55+
recordDiskList(host, token, project, testdataDir)
56+
recordLoopbackAddresses(host, token, testdataDir)
57+
}
58+
59+
func recordTimeseriesQuery(host, token, testdataDir string) {
60+
fmt.Println("Recording timeseries query response...")
61+
62+
body := `{"query": "get hardware_component:voltage | filter slot == 0 && sensor == \"V1P0_MGMT\" | filter timestamp > @now() - 5m | last 5"}`
63+
data, err := doRequest("POST", host+"/v1/system/timeseries/query", token, body)
64+
if err != nil {
65+
log.Printf("Warning: timeseries query failed: %v", err)
66+
return
67+
}
68+
69+
normalized, err := normalizeJSON(data)
70+
if err != nil {
71+
log.Printf("Warning: failed to normalize JSON: %v", err)
72+
return
73+
}
74+
if err := saveFixture(testdataDir, "timeseries_query_response.json", normalized); err != nil {
75+
log.Printf("Warning: %v", err)
76+
return
77+
}
78+
}
79+
80+
func recordDiskList(host, token, project, testdataDir string) {
81+
fmt.Println("Recording disk list response...")
82+
83+
url := fmt.Sprintf("%s/v1/disks?project=%s&limit=5", host, project)
84+
data, err := doRequest("GET", url, token, "")
85+
if err != nil {
86+
log.Printf("Warning: disk list failed: %v", err)
87+
return
88+
}
89+
90+
normalized, err := normalizeJSON(data)
91+
if err != nil {
92+
log.Printf("Warning: failed to normalize JSON: %v", err)
93+
return
94+
}
95+
if err := saveFixture(testdataDir, "disk_list_response.json", normalized); err != nil {
96+
log.Printf("Warning: %v", err)
97+
return
98+
}
99+
}
100+
101+
func recordLoopbackAddresses(host, token, testdataDir string) {
102+
fmt.Println("Recording loopback addresses response...")
103+
104+
url := fmt.Sprintf("%s/v1/system/networking/loopback-address?limit=5", host)
105+
data, err := doRequest("GET", url, token, "")
106+
if err != nil {
107+
log.Printf("Warning: loopback addresses failed: %v", err)
108+
return
109+
}
110+
111+
normalized, err := normalizeJSON(data)
112+
if err != nil {
113+
log.Printf("Warning: failed to normalize JSON: %v", err)
114+
return
115+
}
116+
if err := saveFixture(testdataDir, "loopback_addresses_response.json", normalized); err != nil {
117+
log.Printf("Warning: %v", err)
118+
return
119+
}
120+
}
121+
122+
// doRequest makes a request to the configured nexus instance. We use the standard library here
123+
// and not our own sdk because we're generating test files to verify the generated code.
124+
func doRequest(method, url, token, body string) ([]byte, error) {
125+
var reqBody io.Reader
126+
if body != "" {
127+
reqBody = bytes.NewBufferString(body)
128+
}
129+
130+
req, err := http.NewRequest(method, url, reqBody)
131+
if err != nil {
132+
return nil, fmt.Errorf("failed to create request: %w", err)
133+
}
134+
req.Header.Set("Authorization", "Bearer "+token)
135+
if *apiVersion != "" {
136+
req.Header.Set("API-Version", *apiVersion)
137+
}
138+
if body != "" {
139+
req.Header.Set("Content-Type", "application/json")
140+
}
141+
142+
resp, err := client.Do(req)
143+
if err != nil {
144+
return nil, fmt.Errorf("failed to send request: %w", err)
145+
}
146+
defer resp.Body.Close()
147+
148+
if resp.StatusCode != http.StatusOK {
149+
respBody, _ := io.ReadAll(resp.Body)
150+
return nil, fmt.Errorf("API returned status %d: %s", resp.StatusCode, respBody)
151+
}
152+
153+
return io.ReadAll(resp.Body)
154+
}
155+
156+
func saveFixture(testdataDir, filename string, data []byte) error {
157+
path := filepath.Join(testdataDir, filename)
158+
if err := os.WriteFile(path, data, 0644); err != nil {
159+
return fmt.Errorf("failed to write %s: %w", path, err)
160+
}
161+
return nil
162+
}
163+
164+
// normalizeJSON strips undocumented fields from API responses.
165+
func normalizeJSON(data []byte) ([]byte, error) {
166+
var v any
167+
if err := json.Unmarshal(data, &v); err != nil {
168+
return nil, err
169+
}
170+
171+
// Nexus returns an undocumented `query_summaries` field that's not in the OpenAPI spec. Ignore it for now.
172+
//
173+
// TODO: fully drop `query_summaries` from nexus unless requested.
174+
if m, ok := v.(map[string]any); ok {
175+
delete(m, "query_summaries")
176+
}
177+
178+
return json.Marshal(v)
179+
}
Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
{"items":[{"block_size":512,"description":"boot","device_path":"/mnt/boot","disk_type":"distributed","id":"9caa44a1-2683-4f52-a5ca-8a5ee96c7362","image_id":"14e36227-0984-484e-8c94-baab1a6be648","name":"boot","project_id":"4fb1705d-6f8e-4711-9c38-85fe0fbdc8c1","size":21474836480,"snapshot_id":null,"state":{"state":"detached"},"time_created":"2025-09-25T15:11:27.776013Z","time_modified":"2025-09-25T15:11:27.776013Z"},{"block_size":512,"description":"Created as a boot disk for builder-omni","device_path":"/mnt/builder-omni-omnios-bloody-20250124-d6fb1a","disk_type":"distributed","id":"810c075b-27d0-41b1-b259-7551fed1a004","image_id":"7e7c352c-f4aa-4d8b-a34d-7e7c6eeb615a","name":"builder-omni-omnios-bloody-20250124-d6fb1a","project_id":"4fb1705d-6f8e-4711-9c38-85fe0fbdc8c1","size":1098437885952,"snapshot_id":null,"state":{"state":"detached"},"time_created":"2025-11-27T02:51:41.697116Z","time_modified":"2025-11-27T02:51:41.697116Z"},{"block_size":512,"description":"Created as a boot disk for builder-omni","device_path":"/mnt/builder-omni-omnios-r151056-cloud-228ca2","disk_type":"distributed","id":"2c310fbf-a0bf-4c31-8a3d-a4e38feb53c7","image_id":"6e989035-2319-4980-88a3-31fe7112d87f","name":"builder-omni-omnios-r151056-cloud-228ca2","project_id":"4fb1705d-6f8e-4711-9c38-85fe0fbdc8c1","size":1073741824000,"snapshot_id":null,"state":{"state":"detached"},"time_created":"2025-12-05T20:57:41.635747Z","time_modified":"2025-12-05T20:57:41.635747Z"},{"block_size":512,"description":"Created as a boot disk for ch-builder-omni","device_path":"/mnt/ch-builder-omni-omnios-cloud-1693471113-aadda2","disk_type":"distributed","id":"d1c47f4b-a998-42aa-a273-3b7b87e3e780","image_id":"04ffb229-6c78-4bc4-baa3-b4f07f16bea3","name":"ch-builder-omni-omnios-cloud-1693471113-aadda2","project_id":"4fb1705d-6f8e-4711-9c38-85fe0fbdc8c1","size":1073741824000,"snapshot_id":null,"state":{"state":"detached"},"time_created":"2025-11-27T02:21:32.352141Z","time_modified":"2025-11-27T02:21:32.352141Z"},{"block_size":4096,"description":"data","device_path":"/mnt/data","disk_type":"distributed","id":"84a528d2-b2d5-4420-ae39-3b4643d46a92","image_id":null,"name":"data","project_id":"4fb1705d-6f8e-4711-9c38-85fe0fbdc8c1","size":214748364800,"snapshot_id":null,"state":{"state":"detached"},"time_created":"2025-09-25T15:11:27.728556Z","time_modified":"2025-09-25T15:11:27.728556Z"}],"next_page":"eyJ2IjoidjEiLCJwYWdlX3N0YXJ0Ijp7InNvcnRfYnkiOiJuYW1lX2FzY2VuZGluZyIsInByb2plY3QiOiJjYXJwIiwibGFzdF9zZWVuIjoiZGF0YSJ9fQ=="}
Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
{"items":[{"address":"fd00:99::1/64","address_lot_block_id":"e30373de-88b5-427c-acb7-65f896695e40","id":"887c5912-4bc6-4b6e-a2d6-64d4aa64b5e0","rack_id":"de608e01-b8e4-4d93-b972-a7dbed36dd22","switch_location":"switch0"},{"address":"fd00:99::1/64","address_lot_block_id":"e30373de-88b5-427c-acb7-65f896695e40","id":"b7101671-162a-4a5a-b30e-1bd7696984c5","rack_id":"de608e01-b8e4-4d93-b972-a7dbed36dd22","switch_location":"switch1"}],"next_page":"eyJ2IjoidjEiLCJwYWdlX3N0YXJ0Ijp7InNvcnRfYnkiOiJpZF9hc2NlbmRpbmciLCJsYXN0X3NlZW4iOiJiNzEwMTY3MS0xNjJhLTRhNWEtYjMwZS0xYmQ3Njk2OTg0YzUifX0="}
Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
{"tables":[{"name":"hardware_component:voltage","timeseries":[{"fields":{"chassis_kind":{"type":"string","value":"switch"},"chassis_model":{"type":"string","value":"913-0000006"},"chassis_revision":{"type":"u32","value":4},"chassis_serial":{"type":"string","value":"BRM44220012"},"component_id":{"type":"string","value":"U21"},"component_kind":{"type":"string","value":"tps546b24a"},"description":{"type":"string","value":"V1P0_MGMT rail"},"gateway_id":{"type":"uuid","value":"c0cea24f-ab91-4026-8593-870d64c34673"},"hubris_archive_id":{"type":"string","value":"29806c00ad5fc171"},"rack_id":{"type":"uuid","value":"de608e01-b8e4-4d93-b972-a7dbed36dd22"},"sensor":{"type":"string","value":"V1P0_MGMT"},"slot":{"type":"u32","value":0}},"points":{"start_times":null,"timestamps":["2026-01-12T21:50:24.195794351Z","2026-01-12T21:50:25.192639350Z","2026-01-12T21:50:26.193419069Z","2026-01-12T21:50:27.193322592Z","2026-01-12T21:50:28.290379572Z"],"values":[{"metric_type":"gauge","values":{"type":"double","values":[1.001953125,1.001953125,1.001953125,1.001953125,1.001953125]}}]}},{"fields":{"chassis_kind":{"type":"string","value":"switch"},"chassis_model":{"type":"string","value":"913-0000006"},"chassis_revision":{"type":"u32","value":4},"chassis_serial":{"type":"string","value":"BRM44220012"},"component_id":{"type":"string","value":"U21"},"component_kind":{"type":"string","value":"tps546b24a"},"description":{"type":"string","value":"V1P0_MGMT rail"},"gateway_id":{"type":"uuid","value":"eb645e8f-4228-43fa-9a55-97feabf8ab66"},"hubris_archive_id":{"type":"string","value":"29806c00ad5fc171"},"rack_id":{"type":"uuid","value":"de608e01-b8e4-4d93-b972-a7dbed36dd22"},"sensor":{"type":"string","value":"V1P0_MGMT"},"slot":{"type":"u32","value":0}},"points":{"start_times":null,"timestamps":["2026-01-12T21:50:24.492202137Z","2026-01-12T21:50:25.562686874Z","2026-01-12T21:50:26.493035980Z","2026-01-12T21:50:27.492070298Z","2026-01-12T21:50:28.709615454Z"],"values":[{"metric_type":"gauge","values":{"type":"double","values":[1.001953125,1.001953125,1.001953125,1.001953125,1.001953125]}}]}}]}]}

0 commit comments

Comments
 (0)