Skip to content

Commit aa61b6f

Browse files
jbowensjbowens
committed
rowblk: move 64-bit tests build flag
Move tests that should only run on 64-bit architectures behind a build flag.
1 parent 5242eb1 commit aa61b6f

File tree

2 files changed

+174
-177
lines changed

2 files changed

+174
-177
lines changed

‎sstable/rowblk/rowblk_64bit_test.go‎

Lines changed: 174 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,174 @@
1+
// Copyright 2025 The LevelDB-Go and Pebble Authors. All rights reserved. Use
2+
// of this source code is governed by a BSD-style license that can be found in
3+
// the LICENSE file.
4+
5+
//go:build amd64 || arm64 || arm64be || ppc64 || ppc64le || mips64 || mips64le || s390x || sparc64 || riscv64 || loong64
6+
7+
package rowblk
8+
9+
import (
10+
"bytes"
11+
"fmt"
12+
"math"
13+
"os"
14+
"testing"
15+
16+
"github.com/cockroachdb/errors"
17+
"github.com/cockroachdb/pebble/internal/base"
18+
"github.com/cockroachdb/pebble/internal/buildtags"
19+
"github.com/cockroachdb/pebble/sstable/block"
20+
"github.com/stretchr/testify/require"
21+
)
22+
23+
// TestSingularKVBlockRestartsOverflow tests a scenario where a large key-value
24+
// pair is written to a block, such that the total block size exceeds 4GiB. This
25+
// works becasue the restart table never needs to encode a restart offset beyond
26+
// the 1st key-value pair. The offset of the restarts table itself may exceed
27+
// 2^32-1 but the iterator takes care to support this.
28+
func TestSingularKVBlockRestartsOverflow(t *testing.T) {
29+
_, isCI := os.LookupEnv("CI")
30+
if isCI {
31+
t.Skip("Skipping test: requires too much memory for CI now.")
32+
}
33+
if buildtags.SlowBuild {
34+
t.Skip("Skipping test: requires too much memory for instrumented builds")
35+
}
36+
37+
// Test that SeekGE() and SeekLT() function correctly
38+
// with a singular large KV > 2GB.
39+
40+
const largeKeySize = 2 << 30 // 2GB key size
41+
const largeValueSize = 2 << 30 // 2GB value size
42+
43+
largeKey := bytes.Repeat([]byte("k"), largeKeySize)
44+
largeValue := bytes.Repeat([]byte("v"), largeValueSize)
45+
46+
writer := &Writer{RestartInterval: 1}
47+
require.NoError(t, writer.Add(base.InternalKey{UserKey: largeKey}, largeValue))
48+
blockData := writer.Finish()
49+
iter, err := NewIter(bytes.Compare, nil, nil, blockData, block.NoTransforms)
50+
require.NoError(t, err, "failed to create iterator for block")
51+
52+
// Ensure that SeekGE() does not raise panic due to integer overflow
53+
// indexing problems.
54+
kv := iter.SeekGE(largeKey, base.SeekGEFlagsNone)
55+
56+
// Ensure that SeekGE() finds the correct KV
57+
require.NotNil(t, kv, "failed to find the key")
58+
require.Equal(t, largeKey, kv.K.UserKey, "unexpected key")
59+
require.Equal(t, largeValue, kv.InPlaceValue(), "unexpected value")
60+
61+
// Ensure that SeekGE() does not raise panic due to integer overflow
62+
// indexing problems.
63+
kv = iter.SeekLT([]byte("z"), base.SeekLTFlagsNone)
64+
65+
// Ensure that SeekLT() finds the correct KV
66+
require.NotNil(t, kv, "failed to find the key")
67+
require.Equal(t, largeKey, kv.K.UserKey, "unexpected key")
68+
require.Equal(t, largeValue, kv.InPlaceValue(), "unexpected value")
69+
}
70+
71+
// TestExceedingMaximumRestartOffset tests that writing a block that exceeds the
72+
// maximum restart offset errors.
73+
func TestExceedingMaximumRestartOffset(t *testing.T) {
74+
_, isCI := os.LookupEnv("CI")
75+
if isCI {
76+
t.Skip("Skipping test: requires too much memory for CI now.")
77+
}
78+
if buildtags.SlowBuild {
79+
t.Skip("Skipping test: requires too much memory for instrumented builds")
80+
}
81+
82+
// Test that writing to a block that is already >= 2GiB
83+
// returns an error.
84+
85+
// Adding 512 KVs each with size 4MiB will create a block
86+
// size of >= 2GiB.
87+
const numKVs = 512
88+
const valueSize = (4 << 20)
89+
90+
type KVTestPair struct {
91+
key []byte
92+
value []byte
93+
}
94+
95+
kvTestPairs := make([]KVTestPair, numKVs)
96+
value4MB := bytes.Repeat([]byte("a"), valueSize)
97+
for i := 0; i < numKVs; i++ {
98+
key := fmt.Sprintf("key-%04d", i)
99+
kvTestPairs[i] = KVTestPair{key: []byte(key), value: value4MB}
100+
}
101+
writer := &Writer{RestartInterval: 1}
102+
for _, KVPair := range kvTestPairs {
103+
require.NoError(t, writer.Add(base.InternalKey{UserKey: KVPair.key}, KVPair.value))
104+
}
105+
106+
// Check that buffer is larger than 2GiB.
107+
require.Greater(t, len(writer.buf), MaximumRestartOffset)
108+
109+
// Check that an error is returned after the final write after the 2GiB
110+
// threshold has been crossed
111+
err := writer.Add(base.InternalKey{UserKey: []byte("arbitrary-last-key")}, []byte("arbitrary-last-value"))
112+
require.NotNil(t, err)
113+
require.True(t, errors.Is(err, ErrBlockTooBig))
114+
}
115+
116+
// TestMultipleKVBlockRestartsOverflow tests that SeekGE() works when
117+
// iter.restarts is greater than math.MaxUint32 for multiple KVs. Test writes
118+
// <2GiB to the block and then 4GiB causing iter.restarts to be an int >
119+
// math.MaxUint32. Reaching just shy of 2GiB before adding 4GiB allows the
120+
// final write to succeed without surpassing 2GiB limit. Then verify that
121+
// SeekGE() returns valid output without integer overflow.
122+
//
123+
// Although the block exceeds math.MaxUint32 bytes, no individual KV pair has an
124+
// offset that exceeds MaximumRestartOffset.
125+
func TestMultipleKVBlockRestartsOverflow(t *testing.T) {
126+
if _, isCI := os.LookupEnv("CI"); isCI {
127+
t.Skip("Skipping test: requires too much memory for CI.")
128+
}
129+
if buildtags.SlowBuild {
130+
t.Skip("Skipping test: requires too much memory for instrumented builds")
131+
}
132+
133+
// Write just shy of 2GiB to the block 511 * 4MiB < 2GiB.
134+
const numKVs = 511
135+
const valueSize = 4 * (1 << 20)
136+
const fourGB = 4 * (1 << 30)
137+
138+
type KVTestPair struct {
139+
key []byte
140+
value []byte
141+
}
142+
143+
kvTestPairs := make([]KVTestPair, numKVs)
144+
value4MB := bytes.Repeat([]byte("a"), valueSize)
145+
for i := 0; i < numKVs; i++ {
146+
key := fmt.Sprintf("key-%04d", i)
147+
kvTestPairs[i] = KVTestPair{key: []byte(key), value: value4MB}
148+
}
149+
150+
writer := &Writer{RestartInterval: 1}
151+
for _, KVPair := range kvTestPairs {
152+
writer.Add(base.InternalKey{UserKey: KVPair.key}, KVPair.value)
153+
}
154+
155+
// Add the 4GiB KV, causing iter.restarts >= math.MaxUint32.
156+
// Ensure that SeekGE() works thereafter without integer
157+
// overflows.
158+
writer.Add(base.InternalKey{UserKey: []byte("large-kv")}, bytes.Repeat([]byte("v"), fourGB))
159+
160+
blockData := writer.Finish()
161+
iter, err := NewIter(bytes.Compare, nil, nil, blockData, block.NoTransforms)
162+
require.NoError(t, err, "failed to create iterator for block")
163+
require.Greater(t, int64(iter.restarts), int64(MaximumRestartOffset), "check iter.restarts > 2GiB")
164+
require.Greater(t, int64(iter.restarts), int64(math.MaxUint32), "check iter.restarts > 2^32-1")
165+
166+
for i := 0; i < numKVs; i++ {
167+
key := []byte(fmt.Sprintf("key-%04d", i))
168+
value := bytes.Repeat([]byte("a"), valueSize)
169+
kv := iter.SeekGE(key, base.SeekGEFlagsNone)
170+
require.NotNil(t, kv, "failed to find the large key")
171+
require.Equal(t, key, kv.K.UserKey, "unexpected key")
172+
require.Equal(t, value, kv.InPlaceValue(), "unexpected value")
173+
}
174+
}

‎sstable/rowblk/rowblk_iter_test.go‎

Lines changed: 0 additions & 177 deletions
Original file line numberDiff line numberDiff line change
@@ -7,18 +7,12 @@ package rowblk
77
import (
88
"bytes"
99
"fmt"
10-
"math"
11-
"os"
12-
"runtime"
13-
"strconv"
1410
"strings"
1511
"testing"
1612
"unsafe"
1713

1814
"github.com/cockroachdb/datadriven"
19-
"github.com/cockroachdb/errors"
2015
"github.com/cockroachdb/pebble/internal/base"
21-
"github.com/cockroachdb/pebble/internal/buildtags"
2216
"github.com/cockroachdb/pebble/internal/itertest"
2317
"github.com/cockroachdb/pebble/internal/testkeys"
2418
"github.com/cockroachdb/pebble/sstable/block"
@@ -468,177 +462,6 @@ func TestBlockSyntheticSuffix(t *testing.T) {
468462
}
469463
}
470464

471-
// TestSingularKVBlockRestartsOverflow tests a scenario where a large key-value
472-
// pair is written to a block, such that the total block size exceeds 4GiB. This
473-
// works becasue the restart table never needs to encode a restart offset beyond
474-
// the 1st key-value pair. The offset of the restarts table itself may exceed
475-
// 2^32-1 but the iterator takes care to support this.
476-
func TestSingularKVBlockRestartsOverflow(t *testing.T) {
477-
_, isCI := os.LookupEnv("CI")
478-
if isCI {
479-
t.Skip("Skipping test: requires too much memory for CI now.")
480-
}
481-
if buildtags.SlowBuild {
482-
t.Skip("Skipping test: requires too much memory for instrumented builds")
483-
}
484-
485-
// Test that SeekGE() and SeekLT() function correctly
486-
// with a singular large KV > 2GB.
487-
488-
// Skip this test on 32-bit architectures because they may not
489-
// have sufficient memory to reliably execute this test.
490-
if runtime.GOARCH == "386" || runtime.GOARCH == "arm" || strconv.IntSize == 32 {
491-
t.Skip("Skipping test: not supported on 32-bit architecture")
492-
}
493-
494-
const largeKeySize = 2 << 30 // 2GB key size
495-
const largeValueSize = 2 << 30 // 2GB value size
496-
497-
largeKey := bytes.Repeat([]byte("k"), largeKeySize)
498-
largeValue := bytes.Repeat([]byte("v"), largeValueSize)
499-
500-
writer := &Writer{RestartInterval: 1}
501-
require.NoError(t, writer.Add(base.InternalKey{UserKey: largeKey}, largeValue))
502-
blockData := writer.Finish()
503-
iter, err := NewIter(bytes.Compare, nil, nil, blockData, block.NoTransforms)
504-
require.NoError(t, err, "failed to create iterator for block")
505-
506-
// Ensure that SeekGE() does not raise panic due to integer overflow
507-
// indexing problems.
508-
kv := iter.SeekGE(largeKey, base.SeekGEFlagsNone)
509-
510-
// Ensure that SeekGE() finds the correct KV
511-
require.NotNil(t, kv, "failed to find the key")
512-
require.Equal(t, largeKey, kv.K.UserKey, "unexpected key")
513-
require.Equal(t, largeValue, kv.InPlaceValue(), "unexpected value")
514-
515-
// Ensure that SeekGE() does not raise panic due to integer overflow
516-
// indexing problems.
517-
kv = iter.SeekLT([]byte("z"), base.SeekLTFlagsNone)
518-
519-
// Ensure that SeekLT() finds the correct KV
520-
require.NotNil(t, kv, "failed to find the key")
521-
require.Equal(t, largeKey, kv.K.UserKey, "unexpected key")
522-
require.Equal(t, largeValue, kv.InPlaceValue(), "unexpected value")
523-
}
524-
525-
// TestExceedingMaximumRestartOffset tests that writing a block that exceeds the
526-
// maximum restart offset errors.
527-
func TestExceedingMaximumRestartOffset(t *testing.T) {
528-
_, isCI := os.LookupEnv("CI")
529-
if isCI {
530-
t.Skip("Skipping test: requires too much memory for CI now.")
531-
}
532-
if buildtags.SlowBuild {
533-
t.Skip("Skipping test: requires too much memory for instrumented builds")
534-
}
535-
536-
// Test that writing to a block that is already >= 2GiB
537-
// returns an error.
538-
//
539-
// Skip this test on 32-bit architectures because they may not
540-
// have sufficient memory to reliably execute this test.
541-
if runtime.GOARCH == "386" || runtime.GOARCH == "arm" || strconv.IntSize == 32 {
542-
t.Skip("Skipping test: not supported on 32-bit architecture")
543-
}
544-
545-
// Adding 512 KVs each with size 4MiB will create a block
546-
// size of >= 2GiB.
547-
const numKVs = 512
548-
const valueSize = (4 << 20)
549-
550-
type KVTestPair struct {
551-
key []byte
552-
value []byte
553-
}
554-
555-
KVTestPairs := make([]KVTestPair, numKVs)
556-
value4MB := bytes.Repeat([]byte("a"), valueSize)
557-
for i := 0; i < numKVs; i++ {
558-
key := fmt.Sprintf("key-%04d", i)
559-
KVTestPairs[i] = KVTestPair{key: []byte(key), value: value4MB}
560-
}
561-
writer := &Writer{RestartInterval: 1}
562-
for _, KVPair := range KVTestPairs {
563-
require.NoError(t, writer.Add(base.InternalKey{UserKey: KVPair.key}, KVPair.value))
564-
}
565-
566-
// Check that buffer is larger than 2GiB.
567-
require.Greater(t, len(writer.buf), MaximumRestartOffset)
568-
569-
// Check that an error is returned after the final write after the 2GiB
570-
// threshold has been crossed
571-
err := writer.Add(base.InternalKey{UserKey: []byte("arbitrary-last-key")}, []byte("arbitrary-last-value"))
572-
require.NotNil(t, err)
573-
require.True(t, errors.Is(err, ErrBlockTooBig))
574-
}
575-
576-
// TestMultipleKVBlockRestartsOverflow tests that SeekGE() works when
577-
// iter.restarts is greater than math.MaxUint32 for multiple KVs. Test writes
578-
// <2GiB to the block and then 4GiB causing iter.restarts to be an int >
579-
// math.MaxUint32. Reaching just shy of 2GiB before adding 4GiB allows the
580-
// final write to succeed without surpassing 2GiB limit. Then verify that
581-
// SeekGE() returns valid output without integer overflow.
582-
//
583-
// Although the block exceeds math.MaxUint32 bytes, no individual KV pair has an
584-
// offset that exceeds MaximumRestartOffset.
585-
func TestMultipleKVBlockRestartsOverflow(t *testing.T) {
586-
if _, isCI := os.LookupEnv("CI"); isCI {
587-
t.Skip("Skipping test: requires too much memory for CI.")
588-
}
589-
if buildtags.SlowBuild {
590-
t.Skip("Skipping test: requires too much memory for instrumented builds")
591-
}
592-
593-
// Skip this test on 32-bit architectures because they may not
594-
// have sufficient memory to reliably execute this test.
595-
if runtime.GOARCH == "386" || runtime.GOARCH == "arm" || strconv.IntSize == 32 {
596-
t.Skip("Skipping test: not supported on 32-bit architecture")
597-
}
598-
599-
// Write just shy of 2GiB to the block 511 * 4MiB < 2GiB.
600-
const numKVs = 511
601-
const valueSize = 4 * (1 << 20)
602-
const fourGB = 4 * (1 << 30)
603-
604-
type KVTestPair struct {
605-
key []byte
606-
value []byte
607-
}
608-
609-
KVTestPairs := make([]KVTestPair, numKVs)
610-
value4MB := bytes.Repeat([]byte("a"), valueSize)
611-
for i := 0; i < numKVs; i++ {
612-
key := fmt.Sprintf("key-%04d", i)
613-
KVTestPairs[i] = KVTestPair{key: []byte(key), value: value4MB}
614-
}
615-
616-
writer := &Writer{RestartInterval: 1}
617-
for _, KVPair := range KVTestPairs {
618-
writer.Add(base.InternalKey{UserKey: KVPair.key}, KVPair.value)
619-
}
620-
621-
// Add the 4GiB KV, causing iter.restarts >= math.MaxUint32.
622-
// Ensure that SeekGE() works thereafter without integer
623-
// overflows.
624-
writer.Add(base.InternalKey{UserKey: []byte("large-kv")}, bytes.Repeat([]byte("v"), fourGB))
625-
626-
blockData := writer.Finish()
627-
iter, err := NewIter(bytes.Compare, nil, nil, blockData, block.NoTransforms)
628-
require.NoError(t, err, "failed to create iterator for block")
629-
require.Greater(t, int64(iter.restarts), int64(MaximumRestartOffset), "check iter.restarts > 2GiB")
630-
require.Greater(t, int64(iter.restarts), int64(math.MaxUint32), "check iter.restarts > 2^32-1")
631-
632-
for i := 0; i < numKVs; i++ {
633-
key := []byte(fmt.Sprintf("key-%04d", i))
634-
value := bytes.Repeat([]byte("a"), valueSize)
635-
kv := iter.SeekGE(key, base.SeekGEFlagsNone)
636-
require.NotNil(t, kv, "failed to find the large key")
637-
require.Equal(t, key, kv.K.UserKey, "unexpected key")
638-
require.Equal(t, value, kv.InPlaceValue(), "unexpected value")
639-
}
640-
}
641-
642465
func ikey(s string) base.InternalKey {
643466
return base.InternalKey{UserKey: []byte(s)}
644467
}

0 commit comments

Comments
 (0)