Skip to content

Commit

Permalink
robustness: add function to configure WeightedClusterOptionGroups
Browse files Browse the repository at this point in the history 
Signed-off-by: Siyuan Zhang <sizhang@google.com>
  • Loading branch information
@siyuanfoundation
siyuanfoundation committed May 22, 2024
1 parent 6774334 commit 1566cfe
Show file tree
Hide file tree
Showing 3 changed files with 88 additions and 31 deletions.
44 changes: 44 additions & 0 deletions tests/robustness/options/cluster_options.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
package options

import (
"fmt"
"math/rand"
"time"

Expand All @@ -36,6 +37,21 @@ func WithClusterOptionGroups(input ...ClusterOptions) e2e.EPClusterOption {
}
}

// WithWeightedClusterOptionGroups takes an array of EPClusterOption arrays, and
// randomly picks one EPClusterOption array with given probability distribution when constructing the config.
// This function is mainly used to group strongly coupled config options together, so that we can dynamically test different groups of options.
func WithWeightedClusterOptionGroups(input []ClusterOptions, probs []float64) e2e.EPClusterOption {
if len(input) != len(probs) {
panic("input array size must be equal to probs array size")
}
return func(c *e2e.EtcdProcessClusterConfig) {
optsPicked := input[randIndexWithProbabilities(probs)]
for _, opt := range optsPicked {
opt(c)
}
}
}

// WithSubsetOptions randomly select a subset of input options, and apply the subset to the cluster config.
func WithSubsetOptions(input ...e2e.EPClusterOption) e2e.EPClusterOption {
return func(c *e2e.EtcdProcessClusterConfig) {
Expand All @@ -48,3 +64,31 @@ func WithSubsetOptions(input ...e2e.EPClusterOption) e2e.EPClusterOption {
}
}
}

// randIndexWithProbabilities picks a random index form [0, len(probs)) with probability distribution probs.
func randIndexWithProbabilities(probs []float64) int {
cdf := make([]float64, len(probs))
cum := 0.0

for i := range cdf {
prob := probs[i]
if prob < 0 {
panic(fmt.Sprintf("probability should be >=0, got %f", prob))
}
cum += prob
cdf[i] = cum
}
if cum == 0.0 {
panic("0 cummulative probability")
}
// normalize
for i := range cdf {
cdf[i] /= cum
}
r := internalRand.Float64()
idx := 0
for r > cdf[idx] {
idx++
}
return idx
}
46 changes: 31 additions & 15 deletions tests/robustness/options/cluster_options_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import (
"math/rand"
"testing"

"github.com/stretchr/testify/assert"
"go.etcd.io/etcd/server/v3/embed"
"go.etcd.io/etcd/tests/v3/framework/e2e"
)
Expand All @@ -43,12 +44,12 @@ func TestWithClusterOptionGroups(t *testing.T) {
}

expectedServerConfigs := []embed.Config{
embed.Config{SnapshotCount: 200, SnapshotCatchUpEntries: 100, TickMs: 101, ElectionMs: 1001},
embed.Config{SnapshotCount: 100, SnapshotCatchUpEntries: 100, TickMs: 202, ElectionMs: 2002},
embed.Config{SnapshotCount: 200, SnapshotCatchUpEntries: 100, TickMs: 202, ElectionMs: 2002},
embed.Config{SnapshotCount: 200, SnapshotCatchUpEntries: 100, TickMs: 101, ElectionMs: 1001},
embed.Config{SnapshotCount: 200, SnapshotCatchUpEntries: 100, TickMs: 101, ElectionMs: 1001},
embed.Config{SnapshotCount: 150, SnapshotCatchUpEntries: 100, TickMs: 202, ElectionMs: 2002},
{SnapshotCount: 200, SnapshotCatchUpEntries: 100, TickMs: 101, ElectionMs: 1001},
{SnapshotCount: 100, SnapshotCatchUpEntries: 100, TickMs: 202, ElectionMs: 2002},
{SnapshotCount: 200, SnapshotCatchUpEntries: 100, TickMs: 202, ElectionMs: 2002},
{SnapshotCount: 200, SnapshotCatchUpEntries: 100, TickMs: 101, ElectionMs: 1001},
{SnapshotCount: 200, SnapshotCatchUpEntries: 100, TickMs: 101, ElectionMs: 1001},
{SnapshotCount: 150, SnapshotCatchUpEntries: 100, TickMs: 202, ElectionMs: 2002},
}
for i, tt := range expectedServerConfigs {
cluster := *e2e.NewConfig(opts...)
Expand Down Expand Up @@ -77,20 +78,20 @@ func TestWithOptionsSubset(t *testing.T) {
}

expectedServerConfigs := []embed.Config{
embed.Config{SnapshotCount: 10000, SnapshotCatchUpEntries: 100, TickMs: 100, ElectionMs: 1000},
embed.Config{SnapshotCount: 10000, SnapshotCatchUpEntries: 100, TickMs: 100, ElectionMs: 1000},
embed.Config{SnapshotCount: 10000, SnapshotCatchUpEntries: 100, TickMs: 100, ElectionMs: 1000},
{SnapshotCount: 10000, SnapshotCatchUpEntries: 100, TickMs: 100, ElectionMs: 1000},
{SnapshotCount: 10000, SnapshotCatchUpEntries: 100, TickMs: 100, ElectionMs: 1000},
{SnapshotCount: 10000, SnapshotCatchUpEntries: 100, TickMs: 100, ElectionMs: 1000},
// both SnapshotCount and TickMs&ElectionMs are not default values.
embed.Config{SnapshotCount: 200, SnapshotCatchUpEntries: 100, TickMs: 50, ElectionMs: 500},
embed.Config{SnapshotCount: 10000, SnapshotCatchUpEntries: 100, TickMs: 100, ElectionMs: 1000},
{SnapshotCount: 200, SnapshotCatchUpEntries: 100, TickMs: 50, ElectionMs: 500},
{SnapshotCount: 10000, SnapshotCatchUpEntries: 100, TickMs: 100, ElectionMs: 1000},
// only TickMs&ElectionMs are not default values.
embed.Config{SnapshotCount: 10000, SnapshotCatchUpEntries: 100, TickMs: 50, ElectionMs: 500},
{SnapshotCount: 10000, SnapshotCatchUpEntries: 100, TickMs: 50, ElectionMs: 500},
// both SnapshotCount and TickMs&ElectionMs are not default values.
embed.Config{SnapshotCount: 200, SnapshotCatchUpEntries: 100, TickMs: 50, ElectionMs: 500},
{SnapshotCount: 200, SnapshotCatchUpEntries: 100, TickMs: 50, ElectionMs: 500},
// both SnapshotCount and TickMs&ElectionMs are not default values.
embed.Config{SnapshotCount: 10000, SnapshotCatchUpEntries: 100, TickMs: 50, ElectionMs: 500},
{SnapshotCount: 10000, SnapshotCatchUpEntries: 100, TickMs: 50, ElectionMs: 500},
// only SnapshotCount is not default value.
embed.Config{SnapshotCount: 100, SnapshotCatchUpEntries: 100, TickMs: 100, ElectionMs: 1000},
{SnapshotCount: 100, SnapshotCatchUpEntries: 100, TickMs: 100, ElectionMs: 1000},
}
for i, tt := range expectedServerConfigs {
cluster := *e2e.NewConfig(opts...)
Expand All @@ -108,3 +109,18 @@ func TestWithOptionsSubset(t *testing.T) {
}
}
}

func TestRandIndexWithProbabilities(t *testing.T) {
restore := mockRand(rand.NewSource(1))
defer restore()
probs := []float64{0.3, 0.4, 0.2, 0.1}
samples := make([]float64, len(probs))
sampleSize := 1000
for i := 0; i < sampleSize; i++ {
samples[randIndexWithProbabilities(probs)]++
}
for i := range samples {
samples[i] /= float64(sampleSize)
assert.InDelta(t, samples[i], probs[i], 0.02)
}
}
29 changes: 13 additions & 16 deletions tests/robustness/scenarios.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,22 +69,19 @@ func exploratoryScenarios(_ *testing.T) []testScenario {
options.ClusterOptions{options.WithTickMs(100), options.WithElectionMs(2000)}),
}

mixedVersionOption := options.WithClusterOptionGroups(
// 60% with all members of current version
options.ClusterOptions{options.WithVersion(e2e.CurrentVersion)},
options.ClusterOptions{options.WithVersion(e2e.CurrentVersion)},
options.ClusterOptions{options.WithVersion(e2e.CurrentVersion)},
options.ClusterOptions{options.WithVersion(e2e.CurrentVersion)},
options.ClusterOptions{options.WithVersion(e2e.CurrentVersion)},
options.ClusterOptions{options.WithVersion(e2e.CurrentVersion)},
// 10% with 2 members of current version, 1 member last version, leader is current version
options.ClusterOptions{options.WithVersion(e2e.MinorityLastVersion), options.WithInitialLeaderIndex(0)},
// 10% with 2 members of current version, 1 member last version, leader is last version
options.ClusterOptions{options.WithVersion(e2e.MinorityLastVersion), options.WithInitialLeaderIndex(2)},
// 10% with 2 members of last version, 1 member current version, leader is last version
options.ClusterOptions{options.WithVersion(e2e.QuorumLastVersion), options.WithInitialLeaderIndex(0)},
// 10% with 2 members of last version, 1 member current version, leader is current version
options.ClusterOptions{options.WithVersion(e2e.QuorumLastVersion), options.WithInitialLeaderIndex(2)},
mixedVersionOption := options.WithWeightedClusterOptionGroups(
[]options.ClusterOptions{
// 60% with all members of current version
{options.WithVersion(e2e.CurrentVersion)},
// 10% with 2 members of current version, 1 member last version, leader is current version
{options.WithVersion(e2e.MinorityLastVersion), options.WithInitialLeaderIndex(0)},
// 10% with 2 members of current version, 1 member last version, leader is last version
{options.WithVersion(e2e.MinorityLastVersion), options.WithInitialLeaderIndex(2)},
// 10% with 2 members of last version, 1 member current version, leader is last version
{options.WithVersion(e2e.QuorumLastVersion), options.WithInitialLeaderIndex(0)},
// 10% with 2 members of last version, 1 member current version, leader is current version
{options.WithVersion(e2e.QuorumLastVersion), options.WithInitialLeaderIndex(2)}},
[]float64{0.6, 0.1, 0.1, 0.1, 0.1},
)

baseOptions := []e2e.EPClusterOption{
Expand Down

0 comments on commit 1566cfe

Please sign in to comment.