Skip to content

Commit

Permalink
Fix "State transition loop detected" when retrying status update.
Browse files Browse the repository at this point in the history 
This solves the issue where retrying the status report many times
increases the state loop counter too far and we detect it as a state
loop, even though Mender would actually have finished a few states
later.

This will cause real state loops to take much longer to detect, but
they will *eventually* be detected. Real state loops are quite rare,
since they can only happen either if the system is constantly
rebooting spontaneously, or if an ArtifactVerifyRollbackReboot keeps
returning failure. The spontaneous reboot flow diagram in the Update
Module documentation is a good guide for understanding how it can
happen.

Changelog: Title

Signed-off-by: Kristian Amlie <kristian.amlie@northern.tech>
  • Loading branch information
@kacf
kacf committed Jan 22, 2020
1 parent 3ed4331 commit eb15ff9
Show file tree
Hide file tree
Showing 2 changed files with 185 additions and 3 deletions.
180 changes: 180 additions & 0 deletions app/state_test.go
View file
Expand Up @@ -4162,6 +4162,46 @@ var stateTransitionsWithUpdateModulesTestCases []stateTransitionsWithUpdateModul
&updateRollbackRebootState{},
&updateVerifyRollbackRebootState{},
&updateRollbackRebootState{},
&updateVerifyRollbackRebootState{},
&updateRollbackRebootState{},
&updateVerifyRollbackRebootState{},
&updateRollbackRebootState{},
&updateVerifyRollbackRebootState{},
&updateRollbackRebootState{},
&updateVerifyRollbackRebootState{},
&updateRollbackRebootState{},
&updateVerifyRollbackRebootState{},
&updateRollbackRebootState{},
&updateVerifyRollbackRebootState{},
&updateRollbackRebootState{},
&updateVerifyRollbackRebootState{},
&updateRollbackRebootState{},
&updateVerifyRollbackRebootState{},
&updateRollbackRebootState{},
&updateVerifyRollbackRebootState{},
&updateRollbackRebootState{},
&updateVerifyRollbackRebootState{},
&updateRollbackRebootState{},
&updateVerifyRollbackRebootState{},
&updateRollbackRebootState{},
&updateVerifyRollbackRebootState{},
&updateRollbackRebootState{},
&updateVerifyRollbackRebootState{},
&updateRollbackRebootState{},
&updateVerifyRollbackRebootState{},
&updateRollbackRebootState{},
&updateVerifyRollbackRebootState{},
&updateRollbackRebootState{},
&updateVerifyRollbackRebootState{},
&updateRollbackRebootState{},
&updateVerifyRollbackRebootState{},
&updateRollbackRebootState{},
&updateVerifyRollbackRebootState{},
&updateRollbackRebootState{},
&updateVerifyRollbackRebootState{},
&updateRollbackRebootState{},
&updateVerifyRollbackRebootState{},
&updateRollbackRebootState{},
// Truncated after maximum number of state transitions.
&updateStatusReportState{},
&idleState{},
Expand Down Expand Up @@ -4222,6 +4262,86 @@ var stateTransitionsWithUpdateModulesTestCases []stateTransitionsWithUpdateModul
"ArtifactRollbackReboot",
"ArtifactVerifyRollbackReboot",
"ArtifactRollbackReboot_Leave_00",
"ArtifactRollbackReboot_Enter_00",
"ArtifactRollbackReboot",
"ArtifactVerifyRollbackReboot",
"ArtifactRollbackReboot_Leave_00",
"ArtifactRollbackReboot_Enter_00",
"ArtifactRollbackReboot",
"ArtifactVerifyRollbackReboot",
"ArtifactRollbackReboot_Leave_00",
"ArtifactRollbackReboot_Enter_00",
"ArtifactRollbackReboot",
"ArtifactVerifyRollbackReboot",
"ArtifactRollbackReboot_Leave_00",
"ArtifactRollbackReboot_Enter_00",
"ArtifactRollbackReboot",
"ArtifactVerifyRollbackReboot",
"ArtifactRollbackReboot_Leave_00",
"ArtifactRollbackReboot_Enter_00",
"ArtifactRollbackReboot",
"ArtifactVerifyRollbackReboot",
"ArtifactRollbackReboot_Leave_00",
"ArtifactRollbackReboot_Enter_00",
"ArtifactRollbackReboot",
"ArtifactVerifyRollbackReboot",
"ArtifactRollbackReboot_Leave_00",
"ArtifactRollbackReboot_Enter_00",
"ArtifactRollbackReboot",
"ArtifactVerifyRollbackReboot",
"ArtifactRollbackReboot_Leave_00",
"ArtifactRollbackReboot_Enter_00",
"ArtifactRollbackReboot",
"ArtifactVerifyRollbackReboot",
"ArtifactRollbackReboot_Leave_00",
"ArtifactRollbackReboot_Enter_00",
"ArtifactRollbackReboot",
"ArtifactVerifyRollbackReboot",
"ArtifactRollbackReboot_Leave_00",
"ArtifactRollbackReboot_Enter_00",
"ArtifactRollbackReboot",
"ArtifactVerifyRollbackReboot",
"ArtifactRollbackReboot_Leave_00",
"ArtifactRollbackReboot_Enter_00",
"ArtifactRollbackReboot",
"ArtifactVerifyRollbackReboot",
"ArtifactRollbackReboot_Leave_00",
"ArtifactRollbackReboot_Enter_00",
"ArtifactRollbackReboot",
"ArtifactVerifyRollbackReboot",
"ArtifactRollbackReboot_Leave_00",
"ArtifactRollbackReboot_Enter_00",
"ArtifactRollbackReboot",
"ArtifactVerifyRollbackReboot",
"ArtifactRollbackReboot_Leave_00",
"ArtifactRollbackReboot_Enter_00",
"ArtifactRollbackReboot",
"ArtifactVerifyRollbackReboot",
"ArtifactRollbackReboot_Leave_00",
"ArtifactRollbackReboot_Enter_00",
"ArtifactRollbackReboot",
"ArtifactVerifyRollbackReboot",
"ArtifactRollbackReboot_Leave_00",
"ArtifactRollbackReboot_Enter_00",
"ArtifactRollbackReboot",
"ArtifactVerifyRollbackReboot",
"ArtifactRollbackReboot_Leave_00",
"ArtifactRollbackReboot_Enter_00",
"ArtifactRollbackReboot",
"ArtifactVerifyRollbackReboot",
"ArtifactRollbackReboot_Leave_00",
"ArtifactRollbackReboot_Enter_00",
"ArtifactRollbackReboot",
"ArtifactVerifyRollbackReboot",
"ArtifactRollbackReboot_Leave_00",
"ArtifactRollbackReboot_Enter_00",
"ArtifactRollbackReboot",
"ArtifactVerifyRollbackReboot",
"ArtifactRollbackReboot_Leave_00",
"ArtifactRollbackReboot_Enter_00",
"ArtifactRollbackReboot",
"ArtifactVerifyRollbackReboot",
"ArtifactRollbackReboot_Leave_00",
// Truncated after maximum number of state transitions.
},
reportsLog: []string{
Expand Down Expand Up @@ -4259,6 +4379,26 @@ var stateTransitionsWithUpdateModulesTestCases []stateTransitionsWithUpdateModul
&updateErrorState{},
&updateErrorState{},
&updateErrorState{},
&updateErrorState{},
&updateErrorState{},
&updateErrorState{},
&updateErrorState{},
&updateErrorState{},
&updateErrorState{},
&updateErrorState{},
&updateErrorState{},
&updateErrorState{},
&updateErrorState{},
&updateErrorState{},
&updateErrorState{},
&updateErrorState{},
&updateErrorState{},
&updateErrorState{},
&updateErrorState{},
&updateErrorState{},
&updateErrorState{},
&updateErrorState{},
&updateErrorState{},
// Truncated after maximum number of state transitions.
&updateStatusReportState{},
&idleState{},
Expand Down Expand Up @@ -4301,6 +4441,46 @@ var stateTransitionsWithUpdateModulesTestCases []stateTransitionsWithUpdateModul
"ArtifactFailure",
"ArtifactFailure_Enter_00",
"ArtifactFailure",
"ArtifactFailure_Enter_00",
"ArtifactFailure",
"ArtifactFailure_Enter_00",
"ArtifactFailure",
"ArtifactFailure_Enter_00",
"ArtifactFailure",
"ArtifactFailure_Enter_00",
"ArtifactFailure",
"ArtifactFailure_Enter_00",
"ArtifactFailure",
"ArtifactFailure_Enter_00",
"ArtifactFailure",
"ArtifactFailure_Enter_00",
"ArtifactFailure",
"ArtifactFailure_Enter_00",
"ArtifactFailure",
"ArtifactFailure_Enter_00",
"ArtifactFailure",
"ArtifactFailure_Enter_00",
"ArtifactFailure",
"ArtifactFailure_Enter_00",
"ArtifactFailure",
"ArtifactFailure_Enter_00",
"ArtifactFailure",
"ArtifactFailure_Enter_00",
"ArtifactFailure",
"ArtifactFailure_Enter_00",
"ArtifactFailure",
"ArtifactFailure_Enter_00",
"ArtifactFailure",
"ArtifactFailure_Enter_00",
"ArtifactFailure",
"ArtifactFailure_Enter_00",
"ArtifactFailure",
"ArtifactFailure_Enter_00",
"ArtifactFailure",
"ArtifactFailure_Enter_00",
"ArtifactFailure",
"ArtifactFailure_Enter_00",
"ArtifactFailure",
// Truncated after maximum number of state transitions.
},
reportsLog: []string{
Expand Down
8 changes: 5 additions & 3 deletions datastore/datastore.go
View file
Expand Up @@ -24,10 +24,12 @@ import (
const (
errMsgReadingFromStoreF = "Error reading %q from datastore."

// This number should be kept quite a lot higher than the number of
// This number 30 should be kept quite a lot higher than the number of
// expected state storage operations, which is usually roughly
// equivalent to the number of state transitions.
MaximumStateDataStoreCount = 30
// equivalent to the number of state transitions. 40 is added as an
// extra buffer for StatusReportRetry states, which can run up to 10
// times each (10 * two states * enter and exit state = 10 * 2 * 2 = 40)
MaximumStateDataStoreCount = 30 + 40
)

var (
Expand Down

0 comments on commit eb15ff9

Please sign in to comment.