From ec1abbe85bf9d863418c73a3c76360d6a7b65800 Mon Sep 17 00:00:00 2001 From: Peter Mattis Date: Fri, 11 May 2018 16:56:00 -0400 Subject: [PATCH 1/2] roachtest: make the clock jump and clock monotonic tests subtests Fixes #25432 Release note: None --- pkg/cmd/roachtest/clock_jump_crash.go | 42 ++++++++++++++++++--------- pkg/cmd/roachtest/clock_monotonic.go | 30 ++++++++++--------- pkg/cmd/roachtest/clock_util.go | 23 ++++++++------- pkg/cmd/roachtest/registry.go | 5 ++-- 4 files changed, 59 insertions(+), 41 deletions(-) diff --git a/pkg/cmd/roachtest/clock_jump_crash.go b/pkg/cmd/roachtest/clock_jump_crash.go index 299c8c1afa97..8b1375374acf 100644 --- a/pkg/cmd/roachtest/clock_jump_crash.go +++ b/pkg/cmd/roachtest/clock_jump_crash.go @@ -23,11 +23,7 @@ import ( _ "github.com/lib/pq" ) -func runClockJump(t *test, c *cluster, tc clockJumpTestCase) { - ctx := context.Background() - - c.l.printf("Running %s\n", tc.name) - +func runClockJump(ctx context.Context, t *test, c *cluster, tc clockJumpTestCase) { // Test with a single node so that the node does not crash due to MaxOffset // violation when injecting offset if c.nodes != 1 { @@ -38,8 +34,10 @@ func runClockJump(t *test, c *cluster, tc clockJumpTestCase) { offsetInjector := newOffsetInjector(c) offsetInjector.deploy(ctx) - t.Status("starting cockroach") - c.Put(ctx, cockroach, "./cockroach", c.All()) + if err := c.RunE(ctx, c.Node(1), "test -x ./cockroach"); err != nil { + c.Put(ctx, cockroach, "./cockroach", c.All()) + } + c.Wipe(ctx) c.Start(ctx) db := c.Conn(ctx, c.nodes) @@ -76,9 +74,7 @@ type clockJumpTestCase struct { aliveAfterOffset bool } -func registerClockJump(r *registry) { - const numNodes = 1 - +func makeClockJumpTests() testSpec { testCases := []clockJumpTestCase{ { name: "large_forward_enabled", @@ -112,15 +108,33 @@ func registerClockJump(r *registry) { }, } + spec := testSpec{ + Name: "jump", + Stable: true, // DO NOT COPY to new tests + } + for i := range testCases { tc := testCases[i] - r.Add(testSpec{ - Name: fmt.Sprintf("clockjump/tc=%s", tc.name), - Nodes: nodes(numNodes), + spec.SubTests = append(spec.SubTests, testSpec{ + Name: tc.name, Stable: true, // DO NOT COPY to new tests Run: func(ctx context.Context, t *test, c *cluster) { - runClockJump(t, c, tc) + runClockJump(ctx, t, c, tc) }, }) } + + return spec +} + +func registerClock(r *registry) { + r.Add(testSpec{ + Name: "clock", + Nodes: nodes(1), + Stable: true, // DO NOT COPY to new tests + SubTests: []testSpec{ + makeClockJumpTests(), + makeClockMonotonicTests(), + }, + }) } diff --git a/pkg/cmd/roachtest/clock_monotonic.go b/pkg/cmd/roachtest/clock_monotonic.go index d918a7643abb..17ced22a0a73 100644 --- a/pkg/cmd/roachtest/clock_monotonic.go +++ b/pkg/cmd/roachtest/clock_monotonic.go @@ -23,11 +23,7 @@ import ( _ "github.com/lib/pq" ) -func runClockMonotonicity(t *test, c *cluster, tc clockMonotonicityTestCase) { - ctx := context.Background() - - c.l.printf("Running %s\n", tc.name) - +func runClockMonotonicity(ctx context.Context, t *test, c *cluster, tc clockMonotonicityTestCase) { // Test with a single node so that the node does not crash due to MaxOffset // violation when introducing offset if c.nodes != 1 { @@ -38,8 +34,10 @@ func runClockMonotonicity(t *test, c *cluster, tc clockMonotonicityTestCase) { offsetInjector := newOffsetInjector(c) offsetInjector.deploy(ctx) - t.Status("starting cockroach") - c.Put(ctx, cockroach, "./cockroach", c.All()) + if err := c.RunE(ctx, c.Node(1), "test -x ./cockroach"); err != nil { + c.Put(ctx, cockroach, "./cockroach", c.All()) + } + c.Wipe(ctx) c.Start(ctx) db := c.Conn(ctx, c.nodes) @@ -105,9 +103,7 @@ type clockMonotonicityTestCase struct { expectIncreasingWallTime bool } -func registerClockMonotonicity(r *registry) { - const numNodes = 1 - +func makeClockMonotonicTests() testSpec { testCases := []clockMonotonicityTestCase{ { // Without enabling the feature to persist wall time, wall time is @@ -125,15 +121,21 @@ func registerClockMonotonicity(r *registry) { }, } + spec := testSpec{ + Name: "monotonic", + Stable: true, // DO NOT COPY to new tests + } + for i := range testCases { tc := testCases[i] - r.Add(testSpec{ - Name: fmt.Sprintf("clockmonotonic/tc=%s", tc.name), - Nodes: nodes(numNodes), + spec.SubTests = append(spec.SubTests, testSpec{ + Name: tc.name, Stable: true, // DO NOT COPY to new tests Run: func(ctx context.Context, t *test, c *cluster) { - runClockMonotonicity(t, c, tc) + runClockMonotonicity(ctx, t, c, tc) }, }) } + + return spec } diff --git a/pkg/cmd/roachtest/clock_util.go b/pkg/cmd/roachtest/clock_util.go index efa36b00323f..593728e9fbfb 100644 --- a/pkg/cmd/roachtest/clock_util.go +++ b/pkg/cmd/roachtest/clock_util.go @@ -45,16 +45,19 @@ type offsetInjector struct { // deploy installs ntp and downloads / compiles bumptime used to create a clock offset func (oi *offsetInjector) deploy(ctx context.Context) { - oi.c.Install(ctx, oi.c.All(), "ntp") - oi.c.Install(ctx, oi.c.All(), "gcc") - oi.c.Run(ctx, oi.c.All(), "sudo", "service", "ntp", "stop") - oi.c.Run(ctx, - oi.c.All(), - "curl", - "-kO", - "https://raw.githubusercontent.com/cockroachdb/jepsen/master/cockroachdb/resources/bumptime.c", - ) - oi.c.Run(ctx, oi.c.All(), "gcc", "bumptime.c", "-o", "bumptime", "&&", "rm bumptime.c") + if err := oi.c.RunE(ctx, oi.c.All(), "test -x ./bumptime"); err != nil { + oi.c.Install(ctx, oi.c.All(), "ntp") + oi.c.Install(ctx, oi.c.All(), "gcc") + oi.c.Run(ctx, oi.c.All(), "sudo", "service", "ntp", "stop") + oi.c.Run(ctx, + oi.c.All(), + "curl", + "-kO", + "https://raw.githubusercontent.com/cockroachdb/jepsen/master/cockroachdb/resources/bumptime.c", + ) + oi.c.Run(ctx, oi.c.All(), "gcc", "bumptime.c", "-o", "bumptime", "&&", "rm bumptime.c") + } + oi.deployed = true } diff --git a/pkg/cmd/roachtest/registry.go b/pkg/cmd/roachtest/registry.go index a72621530c66..ab738b628bf8 100644 --- a/pkg/cmd/roachtest/registry.go +++ b/pkg/cmd/roachtest/registry.go @@ -23,16 +23,15 @@ func registerTests(r *registry) { registerAllocator(r) registerBackup(r) registerCancel(r) + registerClock(r) registerCopy(r) registerDebug(r) - registerClockJump(r) - registerClockMonotonicity(r) registerDecommission(r) registerDrop(r) + registerHotSpotSplits(r) registerImportTPCC(r) registerImportTPCH(r) registerJepsen(r) - registerHotSpotSplits(r) registerKV(r) registerKVScalability(r) registerKVSplits(r) From 25361e881c1ffe2d066f091d0f9cfa704925be31 Mon Sep 17 00:00:00 2001 From: Peter Mattis Date: Sat, 12 May 2018 15:11:03 -0400 Subject: [PATCH 2/2] roachtest: deflake clock/jump/small_forward_enabled Fixes #25449 Release note: None --- pkg/cmd/roachtest/clock_jump_crash.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pkg/cmd/roachtest/clock_jump_crash.go b/pkg/cmd/roachtest/clock_jump_crash.go index 8b1375374acf..5c6582175392 100644 --- a/pkg/cmd/roachtest/clock_jump_crash.go +++ b/pkg/cmd/roachtest/clock_jump_crash.go @@ -83,8 +83,11 @@ func makeClockJumpTests() testSpec { aliveAfterOffset: false, }, { - name: "small_forward_enabled", - offset: 150 * time.Millisecond, + name: "small_forward_enabled", + // NB: The offset here needs to be small enough such that this jump plus + // the forward jump check interval (125ms) is less than the tolerated + // forward clock jump (250ms). + offset: 100 * time.Millisecond, jumpCheckEnabled: true, aliveAfterOffset: true, },