Skip to content

Commit 94506e9

Browse files
RaduBerindeclaude
RaduBerinde
and
claude
committed
scripts: add crossversion smoke test
Add a smoke test that validates the crossversion test infrastructure can detect backwards compatibility issues. The test works in three phases: Phase 1: Run crossversion test with clean versions (should pass) Phase 1.5: Run single-version test with patched version (should pass) Phase 2: Run crossversion test with patched version (should fail) The patch introduces an intentional metaindex encoding incompatibility. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
1 parent b4f6e9e commit 94506e9

File tree

2 files changed

+343
-0
lines changed

2 files changed

+343
-0
lines changed

‎scripts/crossversion_smoke_test.patch‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
diff --git a/sstable/layout.go b/sstable/layout.go
2+
index 052295dcf..22deef113 100644
3+
--- a/sstable/layout.go
4+
+++ b/sstable/layout.go
5+
@@ -595,7 +595,9 @@ func decodeLayout(comparer *base.Comparer, data []byte, tableFormat TableFormat)
6+
}
7+
var meta map[string]block.Handle
8+
var vbih valblk.IndexHandle
9+
- if tableFormat >= TableFormatPebblev6 {
10+
+ // INTENTIONAL BUG FOR SMOKE TEST: This introduces a backwards compatibility
11+
+ // issue by changing the metaindex block encoding.
12+
+ if tableFormat < TableFormatPebblev6 {
13+
meta, vbih, err = decodeColumnarMetaIndex(decompressedMeta)
14+
} else {
15+
meta, vbih, err = decodeMetaindex(decompressedMeta)
16+
@@ -1044,7 +1046,7 @@ func (w *layoutWriter) Finish() (size uint64, err error) {
17+
return cmp.Compare(a.key, b.key)
18+
})
19+
var b []byte
20+
- if w.tableFormat >= TableFormatPebblev6 {
21+
+ if w.tableFormat < TableFormatPebblev6 {
22+
var cw colblk.KeyValueBlockWriter
23+
cw.Init()
24+
for _, h := range w.handles {
25+
diff --git a/sstable/reader.go b/sstable/reader.go
26+
index cc17d6283..ba6513e75 100644
27+
--- a/sstable/reader.go
28+
+++ b/sstable/reader.go
29+
@@ -493,7 +493,7 @@ func (r *Reader) readAndDecodeMetaindex(
30+
31+
var meta map[string]block.Handle
32+
var valueBIH valblk.IndexHandle
33+
- if r.tableFormat >= TableFormatPebblev6 {
34+
+ if r.tableFormat < TableFormatPebblev6 {
35+
meta, valueBIH, err = decodeColumnarMetaIndex(data)
36+
} else {
37+
meta, valueBIH, err = decodeMetaindex(data)

‎scripts/crossversion_smoke_test.sh‎

Lines changed: 306 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,306 @@
1+
#!/usr/bin/env bash
2+
3+
# Smoke test for the crossversion test infrastructure.
4+
#
5+
# This script verifies that the crossversion test can detect backwards
6+
# compatibility bugs by:
7+
# 1. Running the crossversion test without any bugs (should PASS)
8+
# 2. Applying a patch that introduces an intentional upgrade bug
9+
# 3. Running the crossversion test again (should FAIL)
10+
# 4. Verifying that the test failed (detecting the bug)
11+
#
12+
# If the crossversion test passes despite the intentional bug, this smoke
13+
# test will fail, alerting us that the crossversion test infrastructure
14+
# may be broken.
15+
16+
set -euo pipefail
17+
18+
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
19+
REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
20+
PATCH_FILE="${SCRIPT_DIR}/crossversion_smoke_test.patch"
21+
CROSSVERSION_DIR="${REPO_ROOT}/internal/metamorphic/crossversion"
22+
23+
# Track whether we applied the patch (for cleanup)
24+
PATCH_APPLIED=false
25+
26+
# Colors for output
27+
RED='\033[0;31m'
28+
GREEN='\033[0;32m'
29+
YELLOW='\033[1;33m'
30+
NC='\033[0m' # No Color
31+
32+
log() {
33+
echo -e "${GREEN}[SMOKE TEST]${NC} $*"
34+
}
35+
36+
warn() {
37+
echo -e "${YELLOW}[SMOKE TEST]${NC} $*"
38+
}
39+
40+
error() {
41+
echo -e "${RED}[SMOKE TEST ERROR]${NC} $*" >&2
42+
}
43+
44+
# Cleanup function
45+
cleanup() {
46+
log "Cleaning up..."
47+
48+
# Remove test binaries
49+
rm -f "${CROSSVERSION_DIR}/release.test"
50+
rm -f "${CROSSVERSION_DIR}/head.test"
51+
rm -f "${CROSSVERSION_DIR}/head-buggy.test"
52+
rm -rf "${CROSSVERSION_DIR}/smoke-test-artifacts"
53+
54+
# Only restore if we applied the patch
55+
if [ "$PATCH_APPLIED" = true ]; then
56+
if ! git diff --quiet; then
57+
warn "Reverting patch changes..."
58+
git checkout -- .
59+
fi
60+
fi
61+
}
62+
63+
# Set up cleanup trap
64+
trap cleanup EXIT
65+
66+
# Check prerequisites
67+
log "Checking prerequisites..."
68+
69+
if [ ! -f "${PATCH_FILE}" ]; then
70+
error "Patch file not found: ${PATCH_FILE}"
71+
exit 1
72+
fi
73+
74+
if ! command -v go >/dev/null 2>&1; then
75+
error "Go is not installed or not in PATH"
76+
exit 1
77+
fi
78+
79+
# Ensure we're in a git repository
80+
if ! git rev-parse --git-dir >/dev/null 2>&1; then
81+
error "Not in a git repository"
82+
exit 1
83+
fi
84+
85+
# Check for uncommitted changes
86+
if ! git diff --quiet || ! git diff --cached --quiet; then
87+
error "Repository has uncommitted changes. Please commit or stash them first."
88+
exit 1
89+
fi
90+
91+
log "Prerequisites OK"
92+
93+
# Step 1: Find the latest release branch
94+
log "Finding latest release branch..."
95+
git fetch origin >/dev/null 2>&1 || warn "Could not fetch from origin (continuing anyway)"
96+
LATEST_RELEASE=$(git branch -r --list '*/crl-release-*' | grep -o 'crl-release-.*$' | sort | tail -1 || echo "")
97+
98+
if [ -z "${LATEST_RELEASE}" ]; then
99+
warn "No release branch found. Trying local branches..."
100+
LATEST_RELEASE=$(git branch --list 'crl-release-*' | sed 's/^[* ]*//' | sort | tail -1 || echo "")
101+
fi
102+
103+
if [ -z "${LATEST_RELEASE}" ]; then
104+
error "Could not find any release branch (crl-release-*)"
105+
error "The crossversion test requires at least one release branch to test against."
106+
exit 1
107+
fi
108+
109+
log "Using release branch: ${LATEST_RELEASE}"
110+
111+
# Step 2: Build test binary from release branch
112+
log "Building test binary from ${LATEST_RELEASE}..."
113+
CURRENT_BRANCH=$(git rev-parse HEAD)
114+
git checkout "${LATEST_RELEASE}" >/dev/null 2>&1
115+
116+
if ! go test -c ./internal/metamorphic -o "${CROSSVERSION_DIR}/release.test"; then
117+
error "Failed to build test binary from ${LATEST_RELEASE}"
118+
git checkout "${CURRENT_BRANCH}" >/dev/null 2>&1
119+
exit 1
120+
fi
121+
122+
git checkout "${CURRENT_BRANCH}" >/dev/null 2>&1
123+
log "Release test binary built successfully"
124+
125+
# Step 3: Build clean HEAD test binary
126+
log "Building clean HEAD test binary..."
127+
if ! go test -c ./internal/metamorphic -o "${CROSSVERSION_DIR}/head.test"; then
128+
error "Failed to build HEAD test binary"
129+
exit 1
130+
fi
131+
log "HEAD test binary built successfully"
132+
133+
# Step 4: Run crossversion test WITHOUT the bug (should PASS)
134+
log ""
135+
log "===== PHASE 1: Running crossversion test WITHOUT bug (should PASS) ====="
136+
log ""
137+
mkdir -p "${CROSSVERSION_DIR}/smoke-test-artifacts"
138+
139+
set +e
140+
echo "::group::Phase 1: Crossversion test WITHOUT bug"
141+
go test -tags invariants -v -timeout 10m -run 'TestMetaCrossVersion' \
142+
./internal/metamorphic/crossversion \
143+
--version "${LATEST_RELEASE},${LATEST_RELEASE},release.test" \
144+
--version 'HEAD,HEAD,head.test' \
145+
--artifacts "${CROSSVERSION_DIR}/smoke-test-artifacts/phase1" \
146+
--seed 42 \
147+
--factor 3 \
148+
2>&1 | tee /tmp/crossversion_smoke_test_phase1.log
149+
150+
PHASE1_EXIT_CODE=$?
151+
echo "::endgroup::"
152+
set -e
153+
154+
if [ ${PHASE1_EXIT_CODE} -ne 0 ]; then
155+
error "SMOKE TEST FAILED!"
156+
error "The crossversion test FAILED without any intentional bug."
157+
error "This indicates there may be an existing compatibility issue."
158+
error "Exit code: ${PHASE1_EXIT_CODE}"
159+
error "Check logs at: /tmp/crossversion_smoke_test_phase1.log"
160+
exit 1
161+
fi
162+
163+
log ""
164+
log "Phase 1 SUCCESS: Crossversion test passed without bug"
165+
log ""
166+
167+
# Step 5: Apply the intentional bug patch
168+
log "===== PHASE 2: Applying intentional bug patch ====="
169+
if ! git apply "${PATCH_FILE}"; then
170+
error "Failed to apply patch. The codebase may have changed."
171+
error "You may need to update the patch file: ${PATCH_FILE}"
172+
exit 1
173+
fi
174+
PATCH_APPLIED=true
175+
log "Patch applied successfully"
176+
177+
# Step 6: Build buggy HEAD test binary
178+
log "Building buggy HEAD test binary..."
179+
if ! go test -c ./internal/metamorphic -o "${CROSSVERSION_DIR}/head-buggy.test"; then
180+
error "Failed to build buggy test binary"
181+
exit 1
182+
fi
183+
log "Buggy test binary built successfully"
184+
185+
# Step 7: Test buggy version by itself (should PASS - verifies patch doesn't break single-version)
186+
log ""
187+
log "===== PHASE 1.5: Testing buggy version by itself (should PASS) ====="
188+
log "This verifies the patch only breaks cross-version compatibility, not single-version."
189+
log ""
190+
191+
set +e
192+
echo "::group::Phase 1.5: Buggy version by itself"
193+
go test -tags invariants -v -timeout 10m -run 'TestMetaCrossVersion' \
194+
./internal/metamorphic/crossversion \
195+
--version 'HEAD-buggy-only,HEAD,head-buggy.test' \
196+
--artifacts "${CROSSVERSION_DIR}/smoke-test-artifacts/phase1.5" \
197+
--seed 42 \
198+
--factor 3 \
199+
2>&1 | tee /tmp/crossversion_smoke_test_phase1.5.log
200+
201+
PHASE1_5_EXIT_CODE=$?
202+
echo "::endgroup::"
203+
set -e
204+
205+
if [ ${PHASE1_5_EXIT_CODE} -ne 0 ]; then
206+
error "SMOKE TEST FAILED!"
207+
error "The buggy version failed when testing itself (single version)."
208+
error "This means the patch introduces a bug that breaks even non-upgrade scenarios."
209+
error "The patch should only break cross-version compatibility, not single-version."
210+
error "Exit code: ${PHASE1_5_EXIT_CODE}"
211+
error "Check logs at: /tmp/crossversion_smoke_test_phase1.5.log"
212+
exit 1
213+
fi
214+
215+
log ""
216+
log "Phase 1.5 SUCCESS: Buggy version works correctly by itself"
217+
log ""
218+
219+
# Step 8: Revert the patch
220+
log "Reverting patch..."
221+
git checkout -- .
222+
PATCH_APPLIED=false
223+
log "Patch reverted"
224+
225+
# Step 9: Run crossversion test WITH the bug multiple times (should FAIL at least once)
226+
log ""
227+
log "===== Running crossversion test WITH bug (should FAIL) ====="
228+
log "Running up to 10 iterations with different seeds to account for randomization..."
229+
log ""
230+
231+
MAX_ATTEMPTS=10
232+
PHASE2_FAILED=false
233+
PHASE2_EXIT_CODE=0
234+
235+
for i in $(seq 1 ${MAX_ATTEMPTS}); do
236+
SEED=$((42 + i))
237+
log "Attempt $i/${MAX_ATTEMPTS} with seed ${SEED}..."
238+
239+
set +e
240+
echo "::group::Phase 2 Attempt $i: Crossversion test WITH bug (seed ${SEED})"
241+
go test -tags invariants -v -timeout 10m -run 'TestMetaCrossVersion' \
242+
./internal/metamorphic/crossversion \
243+
--version "${LATEST_RELEASE},${LATEST_RELEASE},release.test" \
244+
--version 'HEAD-buggy,HEAD,head-buggy.test' \
245+
--artifacts "${CROSSVERSION_DIR}/smoke-test-artifacts/phase2-attempt${i}" \
246+
--seed ${SEED} \
247+
--factor 3 \
248+
2>&1 | tee /tmp/crossversion_smoke_test_phase2_attempt${i}.log
249+
250+
ATTEMPT_EXIT_CODE=$?
251+
echo "::endgroup::"
252+
set -e
253+
254+
if [ ${ATTEMPT_EXIT_CODE} -ne 0 ]; then
255+
log "Attempt $i FAILED (as expected) with exit code: ${ATTEMPT_EXIT_CODE}"
256+
PHASE2_FAILED=true
257+
PHASE2_EXIT_CODE=${ATTEMPT_EXIT_CODE}
258+
break
259+
else
260+
warn "Attempt $i passed (unexpected, but may be due to randomization)"
261+
fi
262+
done
263+
264+
# Step 10: Verify that the test failed in phase 2
265+
log ""
266+
log "===== Checking Phase 2 results ====="
267+
268+
if [ "$PHASE2_FAILED" = false ]; then
269+
error "SMOKE TEST FAILED!"
270+
error "The crossversion test PASSED in all ${MAX_ATTEMPTS} attempts despite the intentional bug."
271+
error "This indicates the crossversion test may not be working correctly."
272+
error "It should have detected the backwards compatibility issue."
273+
error ""
274+
error "Phase 1 (clean cross-version): PASSED (as expected)"
275+
error "Phase 1.5 (buggy single-version): PASSED (as expected)"
276+
error "Phase 2 (buggy cross-version): PASSED in all attempts (UNEXPECTED - should have failed)"
277+
error ""
278+
error "Check logs at: /tmp/crossversion_smoke_test_phase2_attempt*.log"
279+
exit 1
280+
else
281+
log "Phase 2 SUCCESS: Crossversion test correctly detected the intentional bug"
282+
log "Test failed on attempt $i with exit code: ${PHASE2_EXIT_CODE}"
283+
fi
284+
285+
# Show a summary of what was tested
286+
log ""
287+
log "=============================================="
288+
log "SMOKE TEST PASSED!"
289+
log "=============================================="
290+
log ""
291+
log "Summary:"
292+
log " - Release branch: ${LATEST_RELEASE}"
293+
log " - Phase 1 (clean cross-version): PASSED"
294+
log " - Phase 1.5 (buggy single-version): PASSED"
295+
log " - Phase 2 (buggy cross-version): FAILED on attempt $i/$MAX_ATTEMPTS (as expected)"
296+
log " - Intentional bug: Changed metaindex block encoding threshold from v6 to v7"
297+
log ""
298+
log "The crossversion test infrastructure is working correctly and can detect"
299+
log "backwards compatibility issues while not breaking single-version tests."
300+
log ""
301+
log "Logs:"
302+
log " - Phase 1 (clean): /tmp/crossversion_smoke_test_phase1.log"
303+
log " - Phase 1.5 (buggy single): /tmp/crossversion_smoke_test_phase1.5.log"
304+
log " - Phase 2 (buggy cross): /tmp/crossversion_smoke_test_phase2_attempt*.log"
305+
306+
exit 0

0 commit comments

Comments
 (0)