diff --git a/cmd/graphstatemachine/main.go b/cmd/graphstatemachine/main.go index 6cefa4b375b..888b570228e 100644 --- a/cmd/graphstatemachine/main.go +++ b/cmd/graphstatemachine/main.go @@ -35,6 +35,10 @@ func hostStateMachine() stateswitch.StateMachine { func(_ stateswitch.StateSwitch, _ stateswitch.TransitionArgs) error { return nil }, ).AnyTimes() + mockTransitionHandler.EXPECT().PostHostPreparationTimeout().Return( + func(_ stateswitch.StateSwitch, _ stateswitch.TransitionArgs) error { return nil }, + ).AnyTimes() + mockTransitionHandler.EXPECT().PostRefreshLogsProgress(gomock.Any()).Return( func(_ stateswitch.StateSwitch, _ stateswitch.TransitionArgs) error { return nil }, ).AnyTimes() @@ -68,5 +72,9 @@ func poolHostStateMachine() stateswitch.StateMachine { func(_ stateswitch.StateSwitch, _ stateswitch.TransitionArgs) error { return nil }, ).AnyTimes() + mockTransitionHandler.EXPECT().PostHostPreparationTimeout().Return( + func(_ stateswitch.StateSwitch, _ stateswitch.TransitionArgs) error { return nil }, + ).AnyTimes() + return host.NewPoolHostStateMachine(stateswitch.NewStateMachine(), mockTransitionHandler) } diff --git a/cmd/main.go b/cmd/main.go index 342b5dfe582..5df42e430c9 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -313,6 +313,8 @@ func main() { // Make sure that prepare for installation timeout is more than the timeouts of all underlying tools + 2m extra Options.ClusterConfig.PrepareConfig.PrepareForInstallationTimeout = maxDuration(Options.ClusterConfig.PrepareConfig.PrepareForInstallationTimeout, maxDuration(Options.InstructionConfig.DiskCheckTimeout, Options.InstructionConfig.ImageAvailabilityTimeout)+2*time.Minute) + Options.HostConfig.PrepareConfig.PrepareForInstallationTimeout = maxDuration(Options.HostConfig.PrepareConfig.PrepareForInstallationTimeout, + maxDuration(Options.InstructionConfig.DiskCheckTimeout, Options.InstructionConfig.ImageAvailabilityTimeout)+1*time.Minute) var lead leader.ElectorInterface var k8sClient *kubernetes.Clientset var autoMigrationLeader leader.ElectorInterface diff --git a/internal/cluster/statemachine.go b/internal/cluster/statemachine.go index b6f8b29aade..ebc6ba3b059 100644 --- a/internal/cluster/statemachine.go +++ b/internal/cluster/statemachine.go @@ -207,7 +207,7 @@ func NewClusterStateMachine(th TransitionHandler) stateswitch.StateMachine { sm.AddTransitionRule(stateswitch.TransitionRule{ TransitionType: TransitionTypeRefreshStatus, SourceStates: []stateswitch.State{stateswitch.State(models.ClusterStatusPreparingForInstallation)}, - Condition: th.IsPreparingTimedOut, + Condition: stateswitch.And(th.IsPreparingTimedOut, stateswitch.Not(If(FailedPreparingtHostsExist))), DestinationState: stateswitch.State(models.ClusterStatusReady), PostTransition: th.PostPreparingTimedOut, Documentation: stateswitch.TransitionRuleDoc{ diff --git a/internal/host/common.go b/internal/host/common.go index bae4d72b1bd..82d1be9efaf 100644 --- a/internal/host/common.go +++ b/internal/host/common.go @@ -17,12 +17,16 @@ import ( ) const ( - statusInfoMediaDisconnected = "Unable to read from the discovery media. It was either disconnected or poor network conditions prevented it from being read. Try using the minimal ISO option and be sure to keep the media connected until the installation is completed" - statusInfoDisconnected = "Host has stopped communicating with the installation service" - statusInfoDiscovering = "Waiting for host to send hardware details" - statusInfoInsufficientHardware = "Host does not meet the minimum hardware requirements: $FAILING_VALIDATIONS" - statusInfoPendingForInput = "Waiting for user input: $FAILING_VALIDATIONS" - statusInfoNotReadyForInstall = "Host cannot be installed due to following failing validation(s): $FAILING_VALIDATIONS" + statusInfoMediaDisconnected = "Unable to read from the discovery media. It was either disconnected or poor network conditions prevented it from being read. Try using the minimal ISO option and be sure to keep the media connected until the installation is completed" + statusInfoDisconnected = "Host has stopped communicating with the installation service" + statusInfoDiscovering = "Waiting for host to send hardware details" + statusInfoInsufficientHardware = "Host does not meet the minimum hardware requirements: $FAILING_VALIDATIONS" + statusInfoPendingForInput = "Waiting for user input: $FAILING_VALIDATIONS" + statusInfoNotReadyForInstall = "Host cannot be installed due to following failing validation(s): $FAILING_VALIDATIONS" + statusInfoPreparationTimeout = "The host has encountered a preparation timeout, the following conditions failed: $FAILING_CONDITIONS" + statusInfoPreparationTimeoutDiskSpeed = "the installation disk speed check did not complete within the timeout." + statusInfoPreparationTimeoutImageAvailability = "container availability was not determined within the timeout." + statusInfoKnown = "Host is ready to be installed" statusInfoInstalling = "Installation is in progress" statusInfoResettingPendingUserAction = "Host requires booting into the discovery image to complete resetting the installation" diff --git a/internal/host/config.go b/internal/host/config.go index df50f080664..2640587c6d6 100644 --- a/internal/host/config.go +++ b/internal/host/config.go @@ -10,7 +10,12 @@ import ( "github.com/pkg/errors" ) +type PrepareConfig struct { + PrepareForInstallationTimeout time.Duration `envconfig:"PREPARE_FOR_INSTALLATION_HOST_TIMEOUT" default:"8m"` +} + type Config struct { + PrepareConfig PrepareConfig LogTimeoutConfig EnableAutoAssign bool `envconfig:"ENABLE_AUTO_ASSIGN" default:"true"` ResetTimeout time.Duration `envconfig:"RESET_CLUSTER_TIMEOUT" default:"3m"` diff --git a/internal/host/mock_transition.go b/internal/host/mock_transition.go index 6dc43bdb755..1c2524b15a3 100644 --- a/internal/host/mock_transition.go +++ b/internal/host/mock_transition.go @@ -154,6 +154,21 @@ func (mr *MockTransitionHandlerMockRecorder) IsLogCollectionTimedOut(sw, args in return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsLogCollectionTimedOut", reflect.TypeOf((*MockTransitionHandler)(nil).IsLogCollectionTimedOut), sw, args) } +// IsPreparingTimedOut mocks base method. +func (m *MockTransitionHandler) IsPreparingTimedOut(sw stateswitch.StateSwitch, args stateswitch.TransitionArgs) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "IsPreparingTimedOut", sw, args) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// IsPreparingTimedOut indicates an expected call of IsPreparingTimedOut. +func (mr *MockTransitionHandlerMockRecorder) IsPreparingTimedOut(sw, args interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsPreparingTimedOut", reflect.TypeOf((*MockTransitionHandler)(nil).IsPreparingTimedOut), sw, args) +} + // IsUnboundHost mocks base method. func (m *MockTransitionHandler) IsUnboundHost(sw stateswitch.StateSwitch, args stateswitch.TransitionArgs) (bool, error) { m.ctrl.T.Helper() @@ -240,6 +255,20 @@ func (mr *MockTransitionHandlerMockRecorder) PostHostMediaDisconnected(sw, args return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PostHostMediaDisconnected", reflect.TypeOf((*MockTransitionHandler)(nil).PostHostMediaDisconnected), sw, args) } +// PostHostPreparationTimeout mocks base method. +func (m *MockTransitionHandler) PostHostPreparationTimeout() stateswitch.PostTransition { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "PostHostPreparationTimeout") + ret0, _ := ret[0].(stateswitch.PostTransition) + return ret0 +} + +// PostHostPreparationTimeout indicates an expected call of PostHostPreparationTimeout. +func (mr *MockTransitionHandlerMockRecorder) PostHostPreparationTimeout() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PostHostPreparationTimeout", reflect.TypeOf((*MockTransitionHandler)(nil).PostHostPreparationTimeout)) +} + // PostHostProgress mocks base method. func (m *MockTransitionHandler) PostHostProgress(sw stateswitch.StateSwitch, args stateswitch.TransitionArgs) error { m.ctrl.T.Helper() diff --git a/internal/host/monitor.go b/internal/host/monitor.go index 5cf80658408..3be9155cf9b 100644 --- a/internal/host/monitor.go +++ b/internal/host/monitor.go @@ -239,7 +239,6 @@ func (m *Manager) HostMonitoring() { m.log.Debugf("Not a leader, exiting HostMonitoring") return } - m.log.Debugf("Running HostMonitoring") defer commonutils.MeasureOperation("HostMonitoring", m.log, m.metricApi)() m.initMonitoringQueryGenerator() monitored += m.clusterHostMonitoring() diff --git a/internal/host/statemachine.go b/internal/host/statemachine.go index e69b49c3412..a5c6872e343 100644 --- a/internal/host/statemachine.go +++ b/internal/host/statemachine.go @@ -549,6 +549,20 @@ func NewHostStateMachine(sm stateswitch.StateMachine, th TransitionHandler) stat }, }) + sm.AddTransitionRule(stateswitch.TransitionRule{ + TransitionType: TransitionTypeRefresh, + SourceStates: []stateswitch.State{ + stateswitch.State(models.HostStatusPreparingForInstallation), + }, + Condition: stateswitch.And(If(IsConnected), If(IsMediaConnected), th.IsPreparingTimedOut, stateswitch.Or(installationDiskSpeedUnknown, imagesAvailabilityUnknown), allConditionsSuccessfulOrUnknown), + DestinationState: stateswitch.State(models.HostStatusPreparingFailed), + PostTransition: th.PostHostPreparationTimeout(), + Documentation: stateswitch.TransitionRuleDoc{ + Name: "Preparing timed out host move to known", + Description: "TODO: Document this transition rule", + }, + }) + sm.AddTransitionRule(stateswitch.TransitionRule{ TransitionType: TransitionTypeRefresh, SourceStates: []stateswitch.State{ diff --git a/internal/host/transition.go b/internal/host/transition.go index b68ddeb1409..0a934fab269 100644 --- a/internal/host/transition.go +++ b/internal/host/transition.go @@ -60,6 +60,8 @@ type TransitionHandler interface { PostRegisterHost(sw stateswitch.StateSwitch, args stateswitch.TransitionArgs) error PostResettingPendingUserAction(sw stateswitch.StateSwitch, args stateswitch.TransitionArgs) error PostUnbindHost(sw stateswitch.StateSwitch, args stateswitch.TransitionArgs) error + IsPreparingTimedOut(sw stateswitch.StateSwitch, args stateswitch.TransitionArgs) (bool, error) + PostHostPreparationTimeout() stateswitch.PostTransition } var resetLogsField = []interface{}{"logs_info", "", "logs_started_at", strfmt.DateTime(time.Time{}), "logs_collected_at", strfmt.DateTime(time.Time{})} @@ -665,6 +667,37 @@ func (th *transitionHandler) HasInstallationInProgressTimedOut(sw stateswitch.St return time.Since(time.Time(sHost.host.Progress.StageUpdatedAt)) > maxDuration, nil } +func (th *transitionHandler) PostHostPreparationTimeout() stateswitch.PostTransition { + ret := func(sw stateswitch.StateSwitch, args stateswitch.TransitionArgs) error { + sHost, ok := sw.(*stateHost) + if !ok { + return errors.New("PostHostPreparationTimeout incompatible type of StateSwitch") + } + params, ok := args.(*TransitionArgsRefreshHost) + if !ok { + return errors.New("PostRefreshHost invalid argument") + } + var ( + err error + ) + failingConditons := []string{} + if !params.conditions["installation-disk-speed-check-successful"] { + failingConditons = append(failingConditons, statusInfoPreparationTimeoutDiskSpeed) + } + if !params.conditions["successful-container-image-availability"] { + failingConditons = append(failingConditons, statusInfoPreparationTimeoutImageAvailability) + } + statusInfo := strings.Replace(statusInfoPreparationTimeout, "$FAILING_CONDITIONS", strings.Join(failingConditons, "\n"), 1) + if sHost.srcState != swag.StringValue(sHost.host.Status) || swag.StringValue(sHost.host.StatusInfo) != statusInfo { + _, err = hostutil.UpdateHostStatus(params.ctx, logutil.FromContext(params.ctx, th.log), params.db, + th.eventsHandler, sHost.host.InfraEnvID, *sHost.host.ID, + sHost.srcState, swag.StringValue(sHost.host.Status), statusInfo) + } + return err + } + return ret +} + // Return a post transition function with a constant reason func (th *transitionHandler) PostRefreshHost(reason string) stateswitch.PostTransition { ret := func(sw stateswitch.StateSwitch, args stateswitch.TransitionArgs) error { @@ -791,3 +824,19 @@ func (th *transitionHandler) PostRefreshHostRefreshStageUpdateTime( sHost.srcState) return err } + +// check if prepare for installation reach to timeout +func (th *transitionHandler) IsPreparingTimedOut(sw stateswitch.StateSwitch, args stateswitch.TransitionArgs) (bool, error) { + sHost, ok := sw.(*stateHost) + if !ok { + return false, errors.New("IsPreparingTimedOut incompatible type of StateSwitch") + } + // if *sHost.host.Status != models.HostStatusPreparingForInstallation { + // return false, nil + // } + // can happen if the service was rebooted or somehow the async part crashed. + if time.Since(time.Time(sHost.host.StatusUpdatedAt)) > th.config.PrepareConfig.PrepareForInstallationTimeout { + return true, nil + } + return false, nil +} diff --git a/internal/host/transition_test.go b/internal/host/transition_test.go index 16861a4e215..193dc2d639c 100644 --- a/internal/host/transition_test.go +++ b/internal/host/transition_test.go @@ -1327,6 +1327,7 @@ var _ = Describe("Refresh Host", func() { mockVersions := versions.NewMockHandler(ctrl) mockVersions.EXPECT().GetReleaseImage(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). Return(&models.ReleaseImage{URL: swag.String("quay.io/openshift/some-image::latest")}, nil).AnyTimes() + defaultConfig.PrepareConfig.PrepareForInstallationTimeout = 8 * time.Minute hapi = NewManager(common.GetTestLog(), db, testing.GetDummyNotificationStream(ctrl), mockEvents, mockHwValidator, nil, validatorCfg, nil, defaultConfig, nil, operatorsManager, pr, false, nil, mockVersions) hostId = strfmt.UUID(uuid.New().String()) clusterId = strfmt.UUID(uuid.New().String()) @@ -2311,35 +2312,43 @@ var _ = Describe("Refresh Host", func() { // Cluster fields clusterState string + + hostTimedOut bool + validStatusUpdateTime bool }{ { - name: "Preparing no change", - validCheckInTime: true, - dstState: models.HostStatusPreparingForInstallation, - clusterState: models.ClusterStatusPreparingForInstallation, - statusInfoChecker: makeValueChecker(statusInfoPreparingForInstallation), + name: "Preparing no change", + validCheckInTime: true, + validStatusUpdateTime: true, + dstState: models.HostStatusPreparingForInstallation, + clusterState: models.ClusterStatusPreparingForInstallation, + statusInfoChecker: makeValueChecker(statusInfoPreparingForInstallation), validationsChecker: makeJsonChecker(map[validationID]validationCheckResult{ SufficientOrUnknownInstallationDiskSpeed: {status: ValidationSuccess, messagePattern: "Speed of installation disk has not yet been measured"}, SucessfullOrUnknownContainerImagesAvailability: {status: ValidationSuccess, messagePattern: "All required container images were either pulled successfully or no attempt was made to pull them"}, }), + disksInfo: "", + imageStatus: createSuccessfulImageStatuses(), }, { - name: "Cluster not in status", - validCheckInTime: true, - dstState: models.HostStatusKnown, - clusterState: models.ClusterStatusInsufficient, - statusInfoChecker: makeValueChecker(statusInfoKnown), + name: "Cluster not in status", + validCheckInTime: true, + validStatusUpdateTime: true, + dstState: models.HostStatusKnown, + clusterState: models.ClusterStatusInsufficient, + statusInfoChecker: makeValueChecker(statusInfoKnown), validationsChecker: makeJsonChecker(map[validationID]validationCheckResult{ SufficientOrUnknownInstallationDiskSpeed: {status: ValidationSuccess, messagePattern: "Speed of installation disk has not yet been measured"}, SucessfullOrUnknownContainerImagesAvailability: {status: ValidationSuccess, messagePattern: "All required container images were either pulled successfully or no attempt was made to pull them"}, }), }, { - name: "Cluster not in status (2)", - validCheckInTime: true, - dstState: models.HostStatusKnown, - clusterState: models.ClusterStatusInsufficient, - statusInfoChecker: makeValueChecker(statusInfoKnown), + name: "Cluster not in status (2)", + validCheckInTime: true, + validStatusUpdateTime: true, + dstState: models.HostStatusKnown, + clusterState: models.ClusterStatusInsufficient, + statusInfoChecker: makeValueChecker(statusInfoKnown), validationsChecker: makeJsonChecker(map[validationID]validationCheckResult{ SufficientOrUnknownInstallationDiskSpeed: {status: ValidationSuccess, messagePattern: "Speed of installation disk is sufficient"}, SucessfullOrUnknownContainerImagesAvailability: {status: ValidationSuccess, messagePattern: "All required container images were either pulled successfully or no attempt was made to pull them"}, @@ -2347,11 +2356,12 @@ var _ = Describe("Refresh Host", func() { disksInfo: createDiskInfo("/dev/sda", 10, 0), }, { - name: "Disk speed check failed", - validCheckInTime: true, - dstState: models.HostStatusInsufficient, - clusterState: models.ClusterStatusPreparingForInstallation, - statusInfoChecker: makeRegexChecker("Host cannot be installed due to following failing validation.*While preparing the previous installation the installation disk speed measurement failed or was found to be insufficient"), + name: "Disk speed check failed", + validCheckInTime: true, + validStatusUpdateTime: true, + dstState: models.HostStatusInsufficient, + clusterState: models.ClusterStatusPreparingForInstallation, + statusInfoChecker: makeRegexChecker("Host cannot be installed due to following failing validation.*While preparing the previous installation the installation disk speed measurement failed or was found to be insufficient"), validationsChecker: makeJsonChecker(map[validationID]validationCheckResult{ SufficientOrUnknownInstallationDiskSpeed: {status: ValidationFailure, messagePattern: "While preparing the previous installation the installation disk speed measurement failed or was found to be insufficient"}, SucessfullOrUnknownContainerImagesAvailability: {status: ValidationSuccess, messagePattern: "All required container images were either pulled successfully or no attempt was made to pull them"}, @@ -2359,11 +2369,42 @@ var _ = Describe("Refresh Host", func() { disksInfo: createDiskInfo("/dev/sda", 0, -1), }, { - name: "Disk speed check failed after image availability", - validCheckInTime: true, - dstState: models.HostStatusInsufficient, - clusterState: models.ClusterStatusPreparingForInstallation, - statusInfoChecker: makeRegexChecker("Host cannot be installed due to following failing validation.*While preparing the previous installation the installation disk speed measurement failed or was found to be insufficient"), + name: "Disk speed check timed out", + validCheckInTime: true, + validStatusUpdateTime: false, + dstState: models.HostStatusPreparingFailed, + srcState: models.HostStatusPreparingForInstallation, + clusterState: models.ClusterStatusPreparingForInstallation, + statusInfoChecker: makeRegexChecker("The host has encountered a preparation timeout, the following conditions failed: " + statusInfoPreparationTimeoutDiskSpeed), + validationsChecker: makeJsonChecker(map[validationID]validationCheckResult{ + SufficientOrUnknownInstallationDiskSpeed: {status: ValidationSuccess, messagePattern: "Speed of installation disk has not yet been measured"}, + SucessfullOrUnknownContainerImagesAvailability: {status: ValidationSuccess, messagePattern: "All required container images were either pulled successfully or no attempt was made to pull them"}, + }), + disksInfo: "", + imageStatus: createSuccessfulImageStatuses(), + }, + { + name: "Image pull timed out", + validCheckInTime: true, + validStatusUpdateTime: false, + dstState: models.HostStatusPreparingFailed, + srcState: models.HostStatusPreparingForInstallation, + clusterState: models.ClusterStatusPreparingForInstallation, + statusInfoChecker: makeRegexChecker("The host has encountered a preparation timeout, the following conditions failed: " + statusInfoPreparationTimeoutImageAvailability), + validationsChecker: makeJsonChecker(map[validationID]validationCheckResult{ + SufficientOrUnknownInstallationDiskSpeed: {status: ValidationSuccess, messagePattern: "Speed of installation disk is sufficient"}, + SucessfullOrUnknownContainerImagesAvailability: {status: ValidationSuccess, messagePattern: "All required container images were either pulled successfully or no attempt was made to pull them"}, + }), + disksInfo: createDiskInfo("/dev/sda", 10, 0), + imageStatus: "", + }, + { + name: "Disk speed check failed after image availability", + validCheckInTime: true, + validStatusUpdateTime: true, + dstState: models.HostStatusInsufficient, + clusterState: models.ClusterStatusPreparingForInstallation, + statusInfoChecker: makeRegexChecker("Host cannot be installed due to following failing validation.*While preparing the previous installation the installation disk speed measurement failed or was found to be insufficient"), validationsChecker: makeJsonChecker(map[validationID]validationCheckResult{ SufficientOrUnknownInstallationDiskSpeed: {status: ValidationFailure, messagePattern: "While preparing the previous installation the installation disk speed measurement failed or was found to be insufficient"}, SucessfullOrUnknownContainerImagesAvailability: {status: ValidationFailure, messagePattern: "Failed to fetch container images needed for installation from"}, @@ -2372,11 +2413,12 @@ var _ = Describe("Refresh Host", func() { imageStatus: createFailedImageStatuses(), }, { - name: "Image pull failed", - validCheckInTime: true, - dstState: models.HostStatusPreparingFailed, - clusterState: models.ClusterStatusPreparingForInstallation, - statusInfoChecker: makeRegexChecker("Host failed to prepare for installation due to following failing validation.*Failed to fetch container images needed for installation from abc"), + name: "Image pull failed", + validCheckInTime: true, + validStatusUpdateTime: true, + dstState: models.HostStatusPreparingFailed, + clusterState: models.ClusterStatusPreparingForInstallation, + statusInfoChecker: makeRegexChecker("Host failed to prepare for installation due to following failing validation.*Failed to fetch container images needed for installation from abc"), validationsChecker: makeJsonChecker(map[validationID]validationCheckResult{ SufficientOrUnknownInstallationDiskSpeed: {status: ValidationSuccess, messagePattern: "Speed of installation disk has not yet been measured"}, SucessfullOrUnknownContainerImagesAvailability: {status: ValidationFailure, messagePattern: "Failed to fetch container images needed for installation from"}, @@ -2384,12 +2426,13 @@ var _ = Describe("Refresh Host", func() { imageStatus: createFailedImageStatuses(), }, { - name: "Image pull failed and cluster moved to Ready", - validCheckInTime: true, - srcState: models.HostStatusPreparingFailed, - dstState: models.HostStatusKnown, - clusterState: models.ClusterStatusReady, - statusInfoChecker: makeRegexChecker("Host is ready to be installed"), + name: "Image pull failed and cluster moved to Ready", + validCheckInTime: true, + validStatusUpdateTime: true, + srcState: models.HostStatusPreparingFailed, + dstState: models.HostStatusKnown, + clusterState: models.ClusterStatusReady, + statusInfoChecker: makeRegexChecker("Host is ready to be installed"), validationsChecker: makeJsonChecker(map[validationID]validationCheckResult{ SufficientOrUnknownInstallationDiskSpeed: {status: ValidationSuccess, messagePattern: "Speed of installation disk has not yet been measured"}, SucessfullOrUnknownContainerImagesAvailability: {status: ValidationFailure, messagePattern: "Failed to fetch container images needed for installation from"}, @@ -2398,11 +2441,12 @@ var _ = Describe("Refresh Host", func() { }, { - name: "Disk speed check succeeded", - validCheckInTime: true, - dstState: models.HostStatusPreparingForInstallation, - clusterState: models.ClusterStatusPreparingForInstallation, - statusInfoChecker: makeValueChecker(statusInfoPreparingForInstallation), + name: "Disk speed check succeeded", + validCheckInTime: true, + validStatusUpdateTime: true, + dstState: models.HostStatusPreparingForInstallation, + clusterState: models.ClusterStatusPreparingForInstallation, + statusInfoChecker: makeValueChecker(statusInfoPreparingForInstallation), validationsChecker: makeJsonChecker(map[validationID]validationCheckResult{ SufficientOrUnknownInstallationDiskSpeed: {status: ValidationSuccess, messagePattern: "Speed of installation disk is sufficient"}, SucessfullOrUnknownContainerImagesAvailability: {status: ValidationSuccess, messagePattern: "All required container images were either pulled successfully or no attempt was made to pull them"}, @@ -2410,11 +2454,12 @@ var _ = Describe("Refresh Host", func() { disksInfo: createDiskInfo("/dev/sda", 10, 0), }, { - name: "Disk speed should not run on if save partition was set", - validCheckInTime: true, - dstState: models.HostStatusPreparingSuccessful, - clusterState: models.ClusterStatusPreparingForInstallation, - statusInfoChecker: makeValueChecker(statusInfoHostPreparationSuccessful), + name: "Disk speed should not run on if save partition was set", + validCheckInTime: true, + validStatusUpdateTime: true, + dstState: models.HostStatusPreparingSuccessful, + clusterState: models.ClusterStatusPreparingForInstallation, + statusInfoChecker: makeValueChecker(statusInfoHostPreparationSuccessful), validationsChecker: makeJsonChecker(map[validationID]validationCheckResult{ SufficientOrUnknownInstallationDiskSpeed: {status: ValidationSuccess, messagePattern: "Speed of installation disk has not yet been measured"}, SucessfullOrUnknownContainerImagesAvailability: {status: ValidationSuccess, messagePattern: "All required container images were either pulled successfully or no attempt was made to pull them"}, @@ -2423,11 +2468,12 @@ var _ = Describe("Refresh Host", func() { imageStatus: createSuccessfulImageStatuses(), }, { - name: "All succeeded", - validCheckInTime: true, - dstState: models.HostStatusPreparingSuccessful, - clusterState: models.ClusterStatusPreparingForInstallation, - statusInfoChecker: makeValueChecker(statusInfoHostPreparationSuccessful), + name: "All succeeded", + validCheckInTime: true, + validStatusUpdateTime: true, + dstState: models.HostStatusPreparingSuccessful, + clusterState: models.ClusterStatusPreparingForInstallation, + statusInfoChecker: makeValueChecker(statusInfoHostPreparationSuccessful), validationsChecker: makeJsonChecker(map[validationID]validationCheckResult{ SufficientOrUnknownInstallationDiskSpeed: {status: ValidationSuccess, messagePattern: "Speed of installation disk is sufficient"}, SucessfullOrUnknownContainerImagesAvailability: {status: ValidationSuccess, messagePattern: "All required container images were either pulled successfully or no attempt was made to pull them"}, @@ -2447,6 +2493,11 @@ var _ = Describe("Refresh Host", func() { // Timeout for checkin is 3 minutes so subtract 4 minutes from the current time hostCheckInAt = strfmt.DateTime(time.Now().Add(-4 * time.Minute)) } + statusUpdatedAt := strfmt.DateTime(time.Now()) + if !t.validStatusUpdateTime { + // Timeout for checkin is 3 minutes so subtract 4 minutes from the current time + statusUpdatedAt = strfmt.DateTime(time.Now().Add(-9 * time.Minute)) + } srcState := models.HostStatusPreparingForInstallation if t.srcState != "" { srcState = t.srcState @@ -2455,6 +2506,7 @@ var _ = Describe("Refresh Host", func() { host.StatusInfo = swag.String(statusInfoPreparingForInstallation) host.Inventory = hostutil.GenerateMasterInventoryWithHostname("master-0") host.CheckedInAt = hostCheckInAt + host.StatusUpdatedAt = statusUpdatedAt host.DisksInfo = t.disksInfo host.ImagesStatus = t.imageStatus if t.disksInfo == "save_partition" { @@ -4602,6 +4654,7 @@ var _ = Describe("Refresh Host", func() { otherState string otherRequestedHostname string otherInventory string + disksInfo string }{ { name: "insufficient to known", @@ -4627,6 +4680,7 @@ var _ = Describe("Refresh Host", func() { otherState: models.HostStatusInsufficient, otherInventory: hostutil.GenerateMasterInventoryWithHostname("second"), errorExpected: false, + disksInfo: createDiskInfo("/dev/sda", 10, 0), }, { name: "insufficient to insufficient (same hostname) 1", @@ -4656,6 +4710,7 @@ var _ = Describe("Refresh Host", func() { otherState: models.HostStatusInsufficient, otherInventory: hostutil.GenerateMasterInventoryWithHostname("first"), errorExpected: false, + disksInfo: createDiskInfo("/dev/sda", 10, 0), }, { name: "insufficient to insufficient (same hostname) 2", @@ -4686,6 +4741,7 @@ var _ = Describe("Refresh Host", func() { otherInventory: hostutil.GenerateMasterInventoryWithHostname("second"), otherRequestedHostname: "first", errorExpected: false, + disksInfo: createDiskInfo("/dev/sda", 10, 0), }, { name: "insufficient to insufficient (same hostname) 3", @@ -4716,6 +4772,7 @@ var _ = Describe("Refresh Host", func() { otherState: models.HostStatusInsufficient, otherInventory: hostutil.GenerateMasterInventoryWithHostname("second"), errorExpected: false, + disksInfo: createDiskInfo("/dev/sda", 10, 0), }, { name: "insufficient to insufficient (same hostname) 4 loveeee", @@ -4747,6 +4804,7 @@ var _ = Describe("Refresh Host", func() { otherInventory: hostutil.GenerateMasterInventoryWithHostname("second"), otherRequestedHostname: "third", errorExpected: false, + disksInfo: createDiskInfo("/dev/sda", 10, 0), }, { name: "insufficient to known 2", @@ -4777,6 +4835,7 @@ var _ = Describe("Refresh Host", func() { otherInventory: hostutil.GenerateMasterInventoryWithHostname("second"), otherRequestedHostname: "forth", errorExpected: false, + disksInfo: createDiskInfo("/dev/sda", 10, 0), }, { name: "known to known", @@ -4804,6 +4863,7 @@ var _ = Describe("Refresh Host", func() { otherState: models.HostStatusInsufficient, otherInventory: hostutil.GenerateMasterInventoryWithHostname("second"), errorExpected: false, + disksInfo: createDiskInfo("/dev/sda", 10, 0), }, { name: "known to insufficient (same hostname) 1", @@ -4833,6 +4893,7 @@ var _ = Describe("Refresh Host", func() { otherState: models.HostStatusInsufficient, otherInventory: hostutil.GenerateMasterInventoryWithHostname("first"), errorExpected: false, + disksInfo: createDiskInfo("/dev/sda", 10, 0), }, { name: "known to insufficient (same hostname) 2", @@ -4863,6 +4924,7 @@ var _ = Describe("Refresh Host", func() { otherInventory: hostutil.GenerateMasterInventoryWithHostname("second"), otherRequestedHostname: "first", errorExpected: false, + disksInfo: createDiskInfo("/dev/sda", 10, 0), }, { name: "known to insufficient (same hostname) 3", @@ -4893,6 +4955,7 @@ var _ = Describe("Refresh Host", func() { otherState: models.HostStatusInsufficient, otherInventory: hostutil.GenerateMasterInventoryWithHostname("second"), errorExpected: false, + disksInfo: createDiskInfo("/dev/sda", 10, 0), }, { name: "known to insufficient (same hostname) 4", @@ -4924,6 +4987,7 @@ var _ = Describe("Refresh Host", func() { otherInventory: hostutil.GenerateMasterInventoryWithHostname("second"), otherRequestedHostname: "third", errorExpected: false, + disksInfo: createDiskInfo("/dev/sda", 10, 0), }, { name: "known to known 2", @@ -4951,6 +5015,7 @@ var _ = Describe("Refresh Host", func() { otherInventory: hostutil.GenerateMasterInventoryWithHostname("first"), otherRequestedHostname: "forth", errorExpected: false, + disksInfo: createDiskInfo("/dev/sda", 10, 0), }, } @@ -4970,6 +5035,7 @@ var _ = Describe("Refresh Host", func() { bytes, err = json.Marshal(imageStatuses) Expect(err).ShouldNot(HaveOccurred()) host.ImagesStatus = string(bytes) + host.DisksInfo = t.disksInfo domainNameResolutions := common.TestDomainNameResolutionsSuccess bytes, err = json.Marshal(domainNameResolutions) Expect(err).ShouldNot(HaveOccurred()) @@ -6371,6 +6437,10 @@ var _ = Describe("State machine test - refresh transition", func() { func(_ stateswitch.StateSwitch, _ stateswitch.TransitionArgs) error { return nil }, ).AnyTimes() + mockTransitionHandler.EXPECT().PostHostPreparationTimeout().Return( + func(_ stateswitch.StateSwitch, _ stateswitch.TransitionArgs) error { return nil }, + ).AnyTimes() + mockTransitionHandler.EXPECT().PostRefreshLogsProgress(gomock.Any()).Return( func(_ stateswitch.StateSwitch, _ stateswitch.TransitionArgs) error { return nil }, ).AnyTimes() @@ -6393,6 +6463,8 @@ var _ = Describe("State machine test - refresh transition", func() { testState = newTestState(models.HostStatusKnown) BeforeEach(func() { + + mockTransitionHandler.EXPECT().IsPreparingTimedOut(gomock.Any(), gomock.Any()).Return(false, nil).AnyTimes() mockTransitionHandler.EXPECT().PostPreparingForInstallationHost(gomock.Any(), gomock.Any()).Return(nil).AnyTimes() refreshHostArgs = initializeRefreshHostArgs(allValidationIDs, ValidationSuccess, knownStateConditions) @@ -6428,6 +6500,7 @@ var _ = Describe("State machine test - refresh transition", func() { }) It("Moves from known to insufficient when arping-no-ip-collision validation fails", func() { + refreshHostArgs.conditions[string(NoIPCollisionsInNetwork)] = false Expect(stateMachine.Run(TransitionTypeRefresh, testState, &refreshHostArgs)).To(Succeed())