Skip to content

Commit

Permalink
Merge pull request #544 from FactomProject/FD-632_Wait_in_sim_electio…
Browse files Browse the repository at this point in the history 
…n_tests

FD-532 Updated waiting Sim Test to accommodate lagging nodes and worked on runSimTests.sh
  • Loading branch information
@PaulSnow
PaulSnow committed Aug 28, 2018
2 parents c01f76b + 2c1e4d3 commit 1cad4d6
Show file tree
Hide file tree
Showing 2 changed files with 83 additions and 81 deletions.
160 changes: 81 additions & 79 deletions engine/factomd_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@ func SetupSim(GivenNodes string, NetworkType string, UserAddedOptions map[string
state0 := Factomd(params, false).(*state.State)
state0.MessageTally = true
time.Sleep(3 * time.Second)
StatusEveryMinute(state0)
creatingNodes(GivenNodes, state0)

t.Logf("Allocated %d nodes", l)
Expand All @@ -79,8 +80,32 @@ func SetupSim(GivenNodes string, NetworkType string, UserAddedOptions map[string

func creatingNodes(creatingNodes string, state0 *state.State) {
runCmd(fmt.Sprintf("g%d", len(creatingNodes)))
// Wait till all the entries from the g command are processed
simFnodes := GetFnodes()
for {
iq2 := 0
for _, s := range simFnodes {
iq2 += s.State.InMsgQueue2().Length()
}

holding := 0
for _, s := range simFnodes {
iq2 += s.State.InMsgQueue2().Length()
}

pendingCommits := 0
for _, s := range simFnodes {
pendingCommits += s.State.Commits.Len()
}
if iq2 == 0 && pendingCommits == 0 && holding == 0 {
break
}
fmt.Printf("Waiting for g to complete\n")
WaitMinutes(state0, 1)

}
WaitBlocks(state0, 1) // Wait for 1 block
WaitForMinute(state0, 3)
WaitForMinute(state0, 1)
runCmd("0")
for i, c := range []byte(creatingNodes) {
fmt.Println(i)
Expand All @@ -100,6 +125,23 @@ func creatingNodes(creatingNodes string, state0 *state.State) {
WaitForMinute(state0, 1)
}

func WaitForAllNodes(state *state.State) {
height := ""
simFnodes := GetFnodes()
for i := 0; i < len(simFnodes); i++ {
blk := state.LLeaderHeight
s := simFnodes[i].State
height = ""
if s.LLeaderHeight != blk { // if not caught up, start over
time.Sleep(100 * time.Millisecond)
i = 0 // start over
continue
}
height = fmt.Sprintf("%s%s:%d-%d\n", height, s.FactomNodeName, s.LLeaderHeight, s.CurrentMinute)
}
fmt.Printf("Wait for all nodes done\n%s", height)
}

func TimeNow(s *state.State) {
fmt.Printf("%s:%d/%d\n", s.FactomNodeName, int(s.LLeaderHeight), s.CurrentMinute)
}
Expand Down Expand Up @@ -353,7 +395,7 @@ func TestMakeALeader(t *testing.T) {
runCmd("l") // make him a leader
WaitBlocks(state0, 1)
WaitForMinute(state0, 1)

WaitForAllNodes(state0)
CheckAuthoritySet(2, 0, t)
}

Expand Down Expand Up @@ -447,8 +489,9 @@ func TestActivationHeightElection(t *testing.T) {
runCmd("x")
runCmd(fmt.Sprintf("%d", leaders+1))
runCmd("x")
WaitBlocks(state0, 3)
WaitBlocks(state0, 2)
WaitMinutes(state0, 1)
WaitForAllNodes(state0)

if GetFnodes()[leaders].State.Leader {
t.Fatalf("Node %d should not be a leader", leaders)
Expand Down Expand Up @@ -514,7 +557,7 @@ func TestAnElection(t *testing.T) {

for {
pendingCommits := 0
for _, s := range fnodes {
for _, s := range GetFnodes() {
pendingCommits += s.State.Commits.Len()
}
if pendingCommits == 0 {
Expand All @@ -541,8 +584,9 @@ func TestAnElection(t *testing.T) {
//bring him back
runCmd("x")
// wait for him to update via dbstate and become an audit
WaitBlocks(state0, 4)
WaitBlocks(state0, 2)
WaitMinutes(state0, 1)
WaitForAllNodes(state0)

// PrintOneStatus(0, 0)
if GetFnodes()[leaders-1].State.Leader {
Expand Down Expand Up @@ -636,6 +680,8 @@ func TestDBsigEOMElection(t *testing.T) {
runCmd("1")
runCmd("x")
WaitBlocks(state, 2)
WaitForMinute(state, 1)
WaitForAllNodes(state)

CheckAuthoritySet(5, 2, t)

Expand All @@ -653,21 +699,28 @@ func TestMultiple2Election(t *testing.T) {

ranSimTest = true

state0 := SetupSim("LLLLLLLAAF", "LOCAL", map[string]string{}, t)
state := SetupSim("LLLLLLLAAF", "LOCAL", map[string]string{}, t)

CheckAuthoritySet(7, 2, t)

runCmd("1")
runCmd("x")
runCmd("2")
runCmd("x")
WaitForMinute(state, 1)
runCmd("1")
runCmd("x")
runCmd("2")
runCmd("x")

runCmd("s")
runCmd("E")
runCmd("F")
runCmd("0")
runCmd("p")
WaitBlocks(state0, 3)

WaitBlocks(state, 2)
WaitForMinute(state, 1)
WaitForAllNodes(state)

t.Log("Shutting down the network")
for _, fn := range GetFnodes() {
Expand All @@ -682,27 +735,9 @@ func TestMultiple3Election(t *testing.T) {

ranSimTest = true

state0 := SetupSim("LLLLLLLAAAAF", "LOCAL", map[string]string{}, t)
state := SetupSim("LLLLLLLAAAAF", "LOCAL", map[string]string{}, t)

leadercnt := 0
auditcnt := 0
for _, fn := range GetFnodes() {
s := fn.State
if s.Leader {
leadercnt++
}
list := s.ProcessLists.Get(s.LLeaderHeight)
if foundAudit, _ := list.GetAuditServerIndexHash(s.GetIdentityChainID()); foundAudit {
auditcnt++
}
}

if leadercnt != 7 {
t.Fatalf("found %d leaders, expected 7", leadercnt)
}
if auditcnt != 4 {
t.Fatalf("found %d audit, expected 4", auditcnt)
}
CheckAuthoritySet(7, 4, t)

runCmd("1")
runCmd("x")
Expand All @@ -711,35 +746,18 @@ func TestMultiple3Election(t *testing.T) {
runCmd("3")
runCmd("x")
runCmd("0")
WaitMinutes(state0, 1)
WaitMinutes(state, 1)
runCmd("3")
runCmd("x")
runCmd("1")
runCmd("x")
runCmd("2")
runCmd("x")
WaitBlocks(state0, 3)

leadercnt = 0
auditcnt = 0

for _, fn := range GetFnodes() {
s := fn.State
if s.Leader {
leadercnt++
}
list := s.ProcessLists.Get(s.LLeaderHeight)
if foundAudit, _ := list.GetAuditServerIndexHash(s.GetIdentityChainID()); foundAudit {
auditcnt++
}
}
WaitBlocks(state, 2)
WaitForMinute(state, 1)
WaitForAllNodes(state)

if leadercnt != 7 {
t.Fatalf("found %d leaders, expected 7", leadercnt)
}
if auditcnt != 4 {
t.Fatalf("found %d audit, expected 4", auditcnt)
}
CheckAuthoritySet(7, 4, t)

t.Log("Shutting down the network")
for _, fn := range GetFnodes() {
Expand All @@ -755,36 +773,17 @@ func TestMultiple7Election(t *testing.T) {

ranSimTest = true

state0 := SetupSim("LLLLLLLLLLLLLLLAAAAAAAAAA", "LOCAL", map[string]string{"--controlpanelsetting": "readwrite"}, t)

leadercnt := 0
auditcnt := 0
for _, fn := range GetFnodes() {
s := fn.State
if s.Leader {
leadercnt++
}
list := s.ProcessLists.Get(s.LLeaderHeight)
if foundAudit, _ := list.GetAuditServerIndexHash(s.GetIdentityChainID()); foundAudit {
auditcnt++
}
}
state := SetupSim("LLLLLLLLLLLLLLLAAAAAAAAAA", "LOCAL", map[string]string{"--controlpanelsetting": "readwrite"}, t)

if leadercnt != 15 {
t.Fatalf("found %d leaders, expected 15", leadercnt)
}

if auditcnt != 10 {
t.Fatalf("found %d audits, expected 10", auditcnt)
}
CheckAuthoritySet(15, 10, t)

// Take 7 nodes off line
for i := 1; i < 8; i++ {
runCmd(fmt.Sprintf("%d", i))
runCmd("x")
}
// force them all to be faulted
WaitMinutes(state0, 1)
WaitMinutes(state, 1)

// bring them back online
for i := 1; i < 8; i++ {
Expand All @@ -793,7 +792,9 @@ func TestMultiple7Election(t *testing.T) {
}

// Wait till the should have updated by DBSTATE
WaitBlocks(state0, 3)
WaitBlocks(state, 2)
WaitForMinute(state, 1)
WaitForAllNodes(state)

CheckAuthoritySet(15, 10, t)

Expand Down Expand Up @@ -1201,9 +1202,8 @@ func TestDBsigElectionEvery2Block(t *testing.T) {
ranSimTest = true

iterations := 1
state0 := SetupSim("LLLLLLAF", "LOCAL", map[string]string{"--debuglog": "fault|badmsg|network|process|dbsig", "--faulttimeout": "10"}, t)
state := SetupSim("LLLLLLAF", "LOCAL", map[string]string{"--debuglog": "fault|badmsg|network|process|dbsig", "--faulttimeout": "10"}, t)

StatusEveryMinute(state0)
runCmd("S10") // Set Drop Rate to 1.0 on everyone

CheckAuthoritySet(6, 1, t)
Expand All @@ -1222,12 +1222,14 @@ func TestDBsigElectionEvery2Block(t *testing.T) {
}
s.SetNetStateOff(true) // kill the victim
s.LogPrintf("faulting", "Stopped %s\n", s.FactomNodeName)
WaitForMinute(state0, 1) // Wait till FNode0 move ahead a minute (the election is over)
WaitForMinute(state, 1) // Wait till FNode0 move ahead a minute (the election is over)
s.LogPrintf("faulting", "Start %s\n", s.FactomNodeName)
s.SetNetStateOff(false) // resurrect the victim

WaitBlocks(state0, 2) // wait till the victim is back as the audit server
WaitForMinute(state0, 8) // Wait till ablock is loaded
// Wait till the should have updated by DBSTATE
WaitBlocks(state, 2)
WaitForMinute(state, 1)
WaitForAllNodes(state)

CheckAuthoritySet(6, 1, t) // check the authority set is as expected
}
Expand Down Expand Up @@ -1267,7 +1269,7 @@ func TestDBSigElection(t *testing.T) {

WaitBlocks(state0, 2) // wait till the victim is back as the audit server
WaitForMinute(state0, 1) // Wait till ablock is loaded
WaitForMinute(state0, 2) // Wait another couple minute in case any nodes are behind
WaitForAllNodes(state0)

CheckAuthoritySet(3, 1, t) // check the authority set is as expected

Expand Down
4 changes: 2 additions & 2 deletions engine/runSimTest.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,8 @@ mkdir -p test
rm -rf test/*
go test -c github.com/FactomProject/factomd/engine -o test/factomd_test
#run the tests
grep -Eo " Test[^( ]+" factomd_test.go | grep -P "$pattern" | grep -v "TestPass|TestFail|TestRandom"
grep -Eo " Test[^( ]+" factomd_test.go | grep -P "$pattern" | grep -v "TestPass|TestFail|TestRandom"| xargs -I TestMakeALeader -n1 sh -c 'mkdir -p test/TestMakeALeader; cd test/TestMakeALeader; ../factomd_test --test.v --test.timeout 600s --test.run TestMakeALeader 2>&1 | tee testlog.txt'
grep -Eo " Test[^( ]+" factomd_test.go | grep -P "$pattern" | grep -Ev "TestPass|TestFail|TestRandom" | sort
grep -Eo " Test[^( ]+" factomd_test.go | grep -P "$pattern" | grep -Ev "TestPass|TestFail|TestRandom" | sort | xargs -I TestMakeALeader -n1 sh -c 'mkdir -p test/TestMakeALeader; cd test/TestMakeALeader; ../factomd_test --test.v --test.timeout 600s --test.run TestMakeALeader 2>&1 | tee testlog.txt'
find . -name testlog.txt | xargs grep -EH "PASS:|FAIL:|panic"


Expand Down

0 comments on commit 1cad4d6

Please sign in to comment.