Skip to content

Commit

Permalink
Improve IPsec MachineConfig readiness check
Browse files Browse the repository at this point in the history 
Currently CNO checks only machineConfigStatus.Configuration.Source for IPsec
Machine Config are rolled out completely on cluster nodes, but from CI test
with 24 nodes, it's found that source is populated even before UpdatedMachineCount
reaches to MachineCount. So CNO proceeds with rendering IPsec DaemonSet which
makes its pods into crashlooping on those node because ipsec service are not
running yet. Hence this commit also checks MachineCount with UpdatedMachineCount
to ensure IPsec machine config is available on all nodes.

Signed-off-by: Periyasamy Palanisamy <pepalani@redhat.com>
  • Loading branch information
@pperiyasamy
pperiyasamy committed Apr 18, 2024
1 parent b5b41b9 commit 9fcf8e3
Show file tree
Hide file tree
Showing 2 changed files with 19 additions and 16 deletions.
11 changes: 7 additions & 4 deletions pkg/network/ovn_kubernetes.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1461,18 +1461,21 @@ func isIPsecMachineConfigActive(infra bootstrap.InfraStatus) bool {
// One of the IPsec MachineConfig is not created yet, so return false.
return false
}
ipSecPluginOnMasterNodes := hasSourceInMachineConfigStatus(infra.MasterMCPStatus, infra.MasterIPsecMachineConfigs)
ipSecPluginOnWorkerNodes := hasSourceInMachineConfigStatus(infra.WorkerMCPStatus, infra.WorkerIPsecMachineConfigs)
ipSecPluginOnMasterNodes := isIPsecMachineConfigActiveOnPool(infra.MasterMCPStatus, infra.MasterIPsecMachineConfigs)
ipSecPluginOnWorkerNodes := isIPsecMachineConfigActiveOnPool(infra.WorkerMCPStatus, infra.WorkerIPsecMachineConfigs)
return ipSecPluginOnMasterNodes && ipSecPluginOnWorkerNodes
}

func hasSourceInMachineConfigStatus(machineConfigStatus mcfgv1.MachineConfigPoolStatus, machineConfigs []*mcfgv1.MachineConfig) bool {
func isIPsecMachineConfigActiveOnPool(machineConfigStatus mcfgv1.MachineConfigPoolStatus, machineConfigs []*mcfgv1.MachineConfig) bool {
sourceNames := sets.New[string]()
for _, machineConfig := range machineConfigs {
sourceNames.Insert(machineConfig.Name)
}
for _, source := range machineConfigStatus.Configuration.Source {
if sourceNames.Has(source.Name) {
if sourceNames.Has(source.Name) &&
machineConfigStatus.MachineCount == machineConfigStatus.UpdatedMachineCount {
// return true when machine config pool is ready with ipsec plugin. It's okay to skip
// checking other machine configs because that's also rolling out same plugin.
return true
}
}
Expand Down
24 changes: 12 additions & 12 deletions pkg/network/ovn_kubernetes_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2371,9 +2371,9 @@ func TestRenderOVNKubernetesEnableIPsec(t *testing.T) {

// At the 3rd pass, test render logic while MC extension rollout is in progress.
bootstrapResult.Infra.MachineConfigClusterOperatorReady = false
bootstrapResult.Infra.MasterMCPStatus = mcfgv1.MachineConfigPoolStatus{MachineCount: 1, ReadyMachineCount: 0,
bootstrapResult.Infra.MasterMCPStatus = mcfgv1.MachineConfigPoolStatus{MachineCount: 1, ReadyMachineCount: 0, UpdatedMachineCount: 0,
Configuration: mcfgv1.MachineConfigPoolStatusConfiguration{}}
bootstrapResult.Infra.WorkerMCPStatus = mcfgv1.MachineConfigPoolStatus{MachineCount: 1, ReadyMachineCount: 1,
bootstrapResult.Infra.WorkerMCPStatus = mcfgv1.MachineConfigPoolStatus{MachineCount: 1, ReadyMachineCount: 1, UpdatedMachineCount: 1,
Configuration: mcfgv1.MachineConfigPoolStatusConfiguration{}}
objs, _, err = renderOVNKubernetes(config, bootstrapResult, manifestDirOvn, fakeClient, featureGatesCNO)
if err != nil {
Expand Down Expand Up @@ -2402,9 +2402,9 @@ func TestRenderOVNKubernetesEnableIPsec(t *testing.T) {

// At the 4th pass, test render logic once MC extension rollout is complete.
bootstrapResult.Infra.MachineConfigClusterOperatorReady = true
bootstrapResult.Infra.MasterMCPStatus = mcfgv1.MachineConfigPoolStatus{MachineCount: 1, ReadyMachineCount: 1,
bootstrapResult.Infra.MasterMCPStatus = mcfgv1.MachineConfigPoolStatus{MachineCount: 1, ReadyMachineCount: 1, UpdatedMachineCount: 1,
Configuration: mcfgv1.MachineConfigPoolStatusConfiguration{Source: []v1.ObjectReference{{Name: masterMachineConfigIPsecExtName}}}}
bootstrapResult.Infra.WorkerMCPStatus = mcfgv1.MachineConfigPoolStatus{MachineCount: 1, ReadyMachineCount: 1,
bootstrapResult.Infra.WorkerMCPStatus = mcfgv1.MachineConfigPoolStatus{MachineCount: 1, ReadyMachineCount: 1, UpdatedMachineCount: 1,
Configuration: mcfgv1.MachineConfigPoolStatusConfiguration{Source: []v1.ObjectReference{{Name: workerMachineConfigIPsecExtName}}}}
objs, _, err = renderOVNKubernetes(config, bootstrapResult, manifestDirOvn, fakeClient, featureGatesCNO)
if err != nil {
Expand Down Expand Up @@ -2615,9 +2615,9 @@ func TestRenderOVNKubernetesIPsecUpgradeWithMachineConfig(t *testing.T) {
bootstrapResult.Infra.MasterIPsecMachineConfigs[0].Name = masterMachineConfigIPsecExtName
bootstrapResult.Infra.WorkerIPsecMachineConfigs = []*mcfgv1.MachineConfig{{}}
bootstrapResult.Infra.WorkerIPsecMachineConfigs[0].Name = workerMachineConfigIPsecExtName
bootstrapResult.Infra.MasterMCPStatus = mcfgv1.MachineConfigPoolStatus{MachineCount: 1, ReadyMachineCount: 1,
bootstrapResult.Infra.MasterMCPStatus = mcfgv1.MachineConfigPoolStatus{MachineCount: 1, ReadyMachineCount: 1, UpdatedMachineCount: 1,
Configuration: mcfgv1.MachineConfigPoolStatusConfiguration{Source: []v1.ObjectReference{{Name: masterMachineConfigIPsecExtName}}}}
bootstrapResult.Infra.WorkerMCPStatus = mcfgv1.MachineConfigPoolStatus{MachineCount: 1, ReadyMachineCount: 1,
bootstrapResult.Infra.WorkerMCPStatus = mcfgv1.MachineConfigPoolStatus{MachineCount: 1, ReadyMachineCount: 1, UpdatedMachineCount: 1,
Configuration: mcfgv1.MachineConfigPoolStatusConfiguration{Source: []v1.ObjectReference{{Name: workerMachineConfigIPsecExtName}}}}
featureGatesCNO := featuregates.NewFeatureGate([]configv1.FeatureGateName{configv1.FeatureGateAdminNetworkPolicy}, []configv1.FeatureGateName{})
fakeClient := cnofake.NewFakeClient()
Expand Down Expand Up @@ -2779,9 +2779,9 @@ func TestRenderOVNKubernetesIPsecUpgradeWithNoMachineConfig(t *testing.T) {
bootstrapResult.Infra.WorkerIPsecMachineConfigs = []*mcfgv1.MachineConfig{{}}
bootstrapResult.Infra.WorkerIPsecMachineConfigs[0].OwnerReferences = networkOwnerRef()
bootstrapResult.Infra.WorkerIPsecMachineConfigs[0].Name = workerMachineConfigIPsecExtName
bootstrapResult.Infra.MasterMCPStatus = mcfgv1.MachineConfigPoolStatus{MachineCount: 1, ReadyMachineCount: 1,
bootstrapResult.Infra.MasterMCPStatus = mcfgv1.MachineConfigPoolStatus{MachineCount: 1, ReadyMachineCount: 1, UpdatedMachineCount: 1,
Configuration: mcfgv1.MachineConfigPoolStatusConfiguration{Source: []v1.ObjectReference{{Name: masterMachineConfigIPsecExtName}}}}
bootstrapResult.Infra.WorkerMCPStatus = mcfgv1.MachineConfigPoolStatus{MachineCount: 1, ReadyMachineCount: 1,
bootstrapResult.Infra.WorkerMCPStatus = mcfgv1.MachineConfigPoolStatus{MachineCount: 1, ReadyMachineCount: 1, UpdatedMachineCount: 1,
Configuration: mcfgv1.MachineConfigPoolStatusConfiguration{Source: []v1.ObjectReference{{Name: workerMachineConfigIPsecExtName}}}}
objs, _, err = renderOVNKubernetes(config, bootstrapResult, manifestDirOvn, fakeClient, featureGatesCNO)
if err != nil {
Expand Down Expand Up @@ -3001,9 +3001,9 @@ func TestRenderOVNKubernetesDisableIPsec(t *testing.T) {
bootstrapResult.Infra.WorkerIPsecMachineConfigs = []*mcfgv1.MachineConfig{{}}
bootstrapResult.Infra.WorkerIPsecMachineConfigs[0].Name = workerMachineConfigIPsecExtName
bootstrapResult.Infra.WorkerIPsecMachineConfigs[0].OwnerReferences = networkOwnerRef()
bootstrapResult.Infra.MasterMCPStatus = mcfgv1.MachineConfigPoolStatus{MachineCount: 1, ReadyMachineCount: 1,
bootstrapResult.Infra.MasterMCPStatus = mcfgv1.MachineConfigPoolStatus{MachineCount: 1, ReadyMachineCount: 1, UpdatedMachineCount: 1,
Configuration: mcfgv1.MachineConfigPoolStatusConfiguration{Source: []v1.ObjectReference{{Name: masterMachineConfigIPsecExtName}}}}
bootstrapResult.Infra.WorkerMCPStatus = mcfgv1.MachineConfigPoolStatus{MachineCount: 1, ReadyMachineCount: 1,
bootstrapResult.Infra.WorkerMCPStatus = mcfgv1.MachineConfigPoolStatus{MachineCount: 1, ReadyMachineCount: 1, UpdatedMachineCount: 1,
Configuration: mcfgv1.MachineConfigPoolStatusConfiguration{Source: []v1.ObjectReference{{Name: workerMachineConfigIPsecExtName}}}}
objs, _, err := renderOVNKubernetes(config, bootstrapResult, manifestDirOvn, fakeClient, featureGatesCNO)
if err != nil {
Expand Down Expand Up @@ -3132,9 +3132,9 @@ func TestRenderOVNKubernetesDisableIPsecWithUserInstalledIPsecMachineConfigs(t *
bootstrapResult.Infra.MasterIPsecMachineConfigs[0].Name = masterMachineConfigIPsecExtName
bootstrapResult.Infra.WorkerIPsecMachineConfigs = []*mcfgv1.MachineConfig{{}}
bootstrapResult.Infra.WorkerIPsecMachineConfigs[0].Name = workerMachineConfigIPsecExtName
bootstrapResult.Infra.MasterMCPStatus = mcfgv1.MachineConfigPoolStatus{MachineCount: 1, ReadyMachineCount: 1,
bootstrapResult.Infra.MasterMCPStatus = mcfgv1.MachineConfigPoolStatus{MachineCount: 1, ReadyMachineCount: 1, UpdatedMachineCount: 1,
Configuration: mcfgv1.MachineConfigPoolStatusConfiguration{Source: []v1.ObjectReference{{Name: masterMachineConfigIPsecExtName}}}}
bootstrapResult.Infra.WorkerMCPStatus = mcfgv1.MachineConfigPoolStatus{MachineCount: 1, ReadyMachineCount: 1,
bootstrapResult.Infra.WorkerMCPStatus = mcfgv1.MachineConfigPoolStatus{MachineCount: 1, ReadyMachineCount: 1, UpdatedMachineCount: 1,
Configuration: mcfgv1.MachineConfigPoolStatusConfiguration{Source: []v1.ObjectReference{{Name: workerMachineConfigIPsecExtName}}}}
objs, _, err := renderOVNKubernetes(config, bootstrapResult, manifestDirOvn, fakeClient, featureGatesCNO)
if err != nil {
Expand Down

0 comments on commit 9fcf8e3

Please sign in to comment.