Skip to content

Commit

Permalink
operator: refactor and add retry to applyManifests
Browse files Browse the repository at this point in the history
  • Loading branch information
@djoshy
djoshy committed Mar 4, 2024
1 parent a6c488a commit d93d2ce
Showing 1 changed file with 109 additions and 94 deletions.
203 changes: 109 additions & 94 deletions pkg/operator/sync.go
View file
Expand Up @@ -18,7 +18,6 @@ import (
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
"k8s.io/apimachinery/pkg/api/errors"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
Expand All @@ -30,6 +29,7 @@ import (
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/tools/cache"
"k8s.io/client-go/util/retry"
"k8s.io/klog/v2"

configv1 "github.com/openshift/api/config/v1"
Expand Down Expand Up @@ -71,7 +71,6 @@ type manifestPaths struct {
clusterRoleBindings []string
serviceAccounts []string
secrets []string
daemonset string
configMaps []string
roles []string
}
Expand Down Expand Up @@ -348,12 +347,12 @@ func (optr *Operator) syncRenderConfig(_ *renderConfig) error {
}

cm, err = optr.clusterCmLister.ConfigMaps("openshift-config-managed").Get("merged-trusted-image-registry-ca")
if err != nil && !errors.IsNotFound(err) {
if err != nil && !apierrors.IsNotFound(err) {
return err
}
cmAnnotations := make(map[string]string)
cmAnnotations["openshift.io/description"] = "Created and managed by the machine-config-operator"
if err != nil && errors.IsNotFound(err) {
if err != nil && apierrors.IsNotFound(err) {
klog.Infof("creating merged-trusted-image-registry-ca")
_, err = optr.kubeClient.CoreV1().ConfigMaps("openshift-config-managed").Create(
context.TODO(),
Expand Down Expand Up @@ -803,108 +802,100 @@ func (optr *Operator) syncMachineConfigNodes(_ *renderConfig) error {
}
return nil
}

func isApplyManifestErrorRetriable(err error) bool {
// TODO: Update this in case more error filtering is desired in the future
// Putting the following logline to silence verify error
klog.Infof("Checking if %s is a retriable error in isApplyManifestErrorRetriable", err)
return true
}
func (optr *Operator) applyManifests(config *renderConfig, paths manifestPaths) error {
for _, path := range paths.clusterRoles {
crBytes, err := renderAsset(config, path)
if err != nil {
return err
}
cr := resourceread.ReadClusterRoleV1OrDie(crBytes)
_, _, err = resourceapply.ApplyClusterRole(context.TODO(), optr.kubeClient.RbacV1(), optr.libgoRecorder, cr)
if err != nil {
return err
}
}

for _, path := range paths.roles {
rBytes, err := renderAsset(config, path)
if err != nil {
return err
}
r := resourceread.ReadRoleV1OrDie(rBytes)
_, _, err = resourceapply.ApplyRole(context.TODO(), optr.kubeClient.RbacV1(), optr.libgoRecorder, r)
if err != nil {
return err
// Retry manifest apply in case it is a short lived, transient issue as that should not make the Operator Degrade/Unavailable.
return retry.OnError(retry.DefaultRetry, isApplyManifestErrorRetriable, func() error {
for _, path := range paths.clusterRoles {
crBytes, err := renderAsset(config, path)
if err != nil {
return err
}
cr := resourceread.ReadClusterRoleV1OrDie(crBytes)
_, _, err = resourceapply.ApplyClusterRole(context.TODO(), optr.kubeClient.RbacV1(), optr.libgoRecorder, cr)
if err != nil {
return err
}
}
}

for _, path := range paths.roleBindings {
rbBytes, err := renderAsset(config, path)
if err != nil {
return err
}
rb := resourceread.ReadRoleBindingV1OrDie(rbBytes)
_, _, err = resourceapply.ApplyRoleBinding(context.TODO(), optr.kubeClient.RbacV1(), optr.libgoRecorder, rb)
if err != nil {
return err
for _, path := range paths.roles {
rBytes, err := renderAsset(config, path)
if err != nil {
return err
}
r := resourceread.ReadRoleV1OrDie(rBytes)
_, _, err = resourceapply.ApplyRole(context.TODO(), optr.kubeClient.RbacV1(), optr.libgoRecorder, r)
if err != nil {
return err
}
}
}

for _, path := range paths.clusterRoleBindings {
crbBytes, err := renderAsset(config, path)
if err != nil {
return err
}
crb := resourceread.ReadClusterRoleBindingV1OrDie(crbBytes)
_, _, err = resourceapply.ApplyClusterRoleBinding(context.TODO(), optr.kubeClient.RbacV1(), optr.libgoRecorder, crb)
if err != nil {
return err
for _, path := range paths.roleBindings {
rbBytes, err := renderAsset(config, path)
if err != nil {
return err
}
rb := resourceread.ReadRoleBindingV1OrDie(rbBytes)
_, _, err = resourceapply.ApplyRoleBinding(context.TODO(), optr.kubeClient.RbacV1(), optr.libgoRecorder, rb)
if err != nil {
return err
}
}
}

for _, path := range paths.serviceAccounts {
saBytes, err := renderAsset(config, path)
if err != nil {
return err
}
sa := resourceread.ReadServiceAccountV1OrDie(saBytes)
_, _, err = resourceapply.ApplyServiceAccount(context.TODO(), optr.kubeClient.CoreV1(), optr.libgoRecorder, sa)
if err != nil {
return err
for _, path := range paths.clusterRoleBindings {
crbBytes, err := renderAsset(config, path)
if err != nil {
return err
}
crb := resourceread.ReadClusterRoleBindingV1OrDie(crbBytes)
_, _, err = resourceapply.ApplyClusterRoleBinding(context.TODO(), optr.kubeClient.RbacV1(), optr.libgoRecorder, crb)
if err != nil {
return err
}
}
}

for _, path := range paths.secrets {
sBytes, err := renderAsset(config, path)
if err != nil {
return err
}
s := resourceread.ReadSecretV1OrDie(sBytes)
_, _, err = resourceapply.ApplySecret(context.TODO(), optr.kubeClient.CoreV1(), optr.libgoRecorder, s)
if err != nil {
return err
for _, path := range paths.serviceAccounts {
saBytes, err := renderAsset(config, path)
if err != nil {
return err
}
sa := resourceread.ReadServiceAccountV1OrDie(saBytes)
_, _, err = resourceapply.ApplyServiceAccount(context.TODO(), optr.kubeClient.CoreV1(), optr.libgoRecorder, sa)
if err != nil {
return err
}
}
}

for _, path := range paths.configMaps {
cmBytes, err := renderAsset(config, path)
if err != nil {
return err
}
cm := resourceread.ReadConfigMapV1OrDie(cmBytes)
_, _, err = resourceapply.ApplyConfigMap(context.TODO(), optr.kubeClient.CoreV1(), optr.libgoRecorder, cm)
if err != nil {
return err
for _, path := range paths.secrets {
sBytes, err := renderAsset(config, path)
if err != nil {
return err
}
s := resourceread.ReadSecretV1OrDie(sBytes)
_, _, err = resourceapply.ApplySecret(context.TODO(), optr.kubeClient.CoreV1(), optr.libgoRecorder, s)
if err != nil {
return err
}
}
}

if paths.daemonset != "" {
dBytes, err := renderAsset(config, paths.daemonset)
if err != nil {
return err
}
d := resourceread.ReadDaemonSetV1OrDie(dBytes)
_, updated, err := mcoResourceApply.ApplyDaemonSet(optr.kubeClient.AppsV1(), d)
if err != nil {
return err
}
if updated {
return optr.waitForDaemonsetRollout(d)
for _, path := range paths.configMaps {
cmBytes, err := renderAsset(config, path)
if err != nil {
return err
}
cm := resourceread.ReadConfigMapV1OrDie(cmBytes)
_, _, err = resourceapply.ApplyConfigMap(context.TODO(), optr.kubeClient.CoreV1(), optr.libgoRecorder, cm)
if err != nil {
return err
}
}
}

return nil
return nil
})
}

// safetySyncControllerConfig is a special case render of the controllerconfig that we run when
Expand Down Expand Up @@ -1221,7 +1212,6 @@ func (optr *Operator) syncMachineConfigDaemon(config *renderConfig) error {
serviceAccounts: []string{
mcdServiceAccountManifestPath,
},
daemonset: mcdDaemonsetManifestPath,
configMaps: []string{
mcdKubeRbacProxyConfigMapPath,
},
Expand All @@ -1231,6 +1221,19 @@ func (optr *Operator) syncMachineConfigDaemon(config *renderConfig) error {
return fmt.Errorf("failed to apply machine config daemon manifests: %w", err)
}

dBytes, err := renderAsset(config, mcdDaemonsetManifestPath)
if err != nil {
return err
}
d := resourceread.ReadDaemonSetV1OrDie(dBytes)
_, updated, err := mcoResourceApply.ApplyDaemonSet(optr.kubeClient.AppsV1(), d)
if err != nil {
return err
}
if updated {
return optr.waitForDaemonsetRollout(d)
}

return nil
}

Expand All @@ -1251,13 +1254,25 @@ func (optr *Operator) syncMachineConfigServer(config *renderConfig) error {
secrets: []string{
mcsNodeBootstrapperTokenManifestPath,
},
daemonset: mcsDaemonsetManifestPath,
}

if err := optr.applyManifests(config, paths); err != nil {
return fmt.Errorf("failed to apply machine config server manifests: %w", err)
}

dBytes, err := renderAsset(config, mcsDaemonsetManifestPath)
if err != nil {
return err
}
d := resourceread.ReadDaemonSetV1OrDie(dBytes)
_, updated, err := mcoResourceApply.ApplyDaemonSet(optr.kubeClient.AppsV1(), d)
if err != nil {
return err
}
if updated {
return optr.waitForDaemonsetRollout(d)
}

return nil
}

Expand Down

0 comments on commit d93d2ce

Please sign in to comment.