Fix(reconciliation) Rework reconciliation loop. Close #131.

* Use server side apply instead of client side apply (https://kubernetes.io/docs/reference/using-api/server-side-apply/) Remove pkg/resources/mutation package Add mandatory protocol for ports * Fix deployment check status function Add more tests deployment resource * Fix service and secret check status * Fix processFunc using dependency management * Never bypass reconciliation loop * Rename withStatus to onlySpec to ignore metadata fields * Fix JSON Marshall for all harbor apis/ Signed-off-by: Simon Guyennet <simon.guyennet@corp.ovh.com> Signed-off-by: Pierre Peronnet <pierre.peronnet@ovhcloud.com>
goharbor · Jan 18, 2021 · 0d3cdd1 · 0d3cdd1
1 parent d3cc520
commit 0d3cdd1
Show file tree

Hide file tree

Showing 56 changed files with 959 additions and 946 deletions.
diff --git a/Makefile b/Makefile
@@ -431,7 +431,8 @@ INGRESS_NAMESPACE := nginx-ingress
 .PHONY: ingress
 ingress: helm
 	$(MAKE) kube-namespace NAMESPACE=$(INGRESS_NAMESPACE)
-	$(HELM) upgrade --install nginx stable/nginx-ingress \
+	$(HELM) repo add ingress-nginx https://kubernetes.github.io/ingress-nginx # https://github.com/kubernetes/ingress-nginx/tree/master/charts/ingress-nginx#get-repo-info
+	$(HELM) upgrade --install nginx ingress-nginx/ingress-nginx \
 		--namespace $(INGRESS_NAMESPACE) \
 		--set-string controller.config.proxy-body-size=0
 
@@ -440,6 +441,7 @@ CERTMANAGER_NAMESPACE := cert-manager
 .PHONY: certmanager
 certmanager: helm jetstack
 	$(MAKE) kube-namespace NAMESPACE=$(CERTMANAGER_NAMESPACE)
+	$(HELM) repo add jetstack https://charts.jetstack.io # https://cert-manager.io/docs/installation/kubernetes/
 	$(HELM) upgrade --install certmanager jetstack/cert-manager \
 		--namespace $(CERTMANAGER_NAMESPACE) \
 		--version v0.15.1 \

diff --git a/apis/meta/v1alpha1/component.go b/apis/meta/v1alpha1/component.go
@@ -118,7 +118,7 @@ type ComponentStatus struct {
 	// Current number of pods.
 	// +kubebuilder:validation:Optional
 	// +kubebuilder:validation:Minimum=0
-	Replicas *int32 `json:"replicas"`
+	Replicas *int32 `json:"replicas,omitempty"`
 
 	// Conditions list of extracted conditions from Resource
 	// +listType:map
@@ -128,12 +128,14 @@ type ComponentStatus struct {
 
 func (s ComponentStatus) MarshalJSON() ([]byte, error) {
 	var data struct {
-		ObservedGeneration int64         `json:"observedGeneration,omitempty"`
-		Status             status.Status `json:"status"`
-		Message            string        `json:"message"`
-		Conditions         []Condition   `json:"conditions"`
+		ObservedGeneration int64          `json:"observedGeneration,omitempty"`
+		Operator           OperatorStatus `json:"operator,omitempty"`
+		Replicas           *int32         `json:"replicas,omitempty"`
+		Conditions         []Condition    `json:"conditions"`
 	}
 
+	data.Operator = s.Operator
+	data.Replicas = s.Replicas
 	data.ObservedGeneration = s.ObservedGeneration
 
 	if s.Conditions == nil {

diff --git a/controllers/goharbor/chartmuseum/deployments.go b/controllers/goharbor/chartmuseum/deployments.go
@@ -301,9 +301,11 @@ func (r *Reconciler) GetDeployment(ctx context.Context, chartMuseum *goharborv1a
 						Ports: []corev1.ContainerPort{{
 							Name:          harbormetav1.ChartMuseumHTTPPortName,
 							ContainerPort: httpPort,
+							Protocol:      corev1.ProtocolTCP,
 						}, {
 							Name:          harbormetav1.ChartMuseumHTTPSPortName,
 							ContainerPort: httpsPort,
+							Protocol:      corev1.ProtocolTCP,
 						}},
 
 						EnvFrom: envFroms,

diff --git a/controllers/goharbor/chartmuseum/services.go b/controllers/goharbor/chartmuseum/services.go
@@ -24,10 +24,12 @@ func (r *Reconciler) GetService(ctx context.Context, chartMuseum *goharborv1alph
 				Name:       harbormetav1.ChartMuseumHTTPPortName,
 				Port:       harbormetav1.HTTPPort,
 				TargetPort: intstr.FromString(harbormetav1.ChartMuseumHTTPPortName),
+				Protocol:   corev1.ProtocolTCP,
 			}, {
 				Name:       harbormetav1.ChartMuseumHTTPSPortName,
 				Port:       harbormetav1.HTTPSPort,
 				TargetPort: intstr.FromString(harbormetav1.ChartMuseumHTTPSPortName),
+				Protocol:   corev1.ProtocolTCP,
 			}},
 			Selector: map[string]string{
 				r.Label("name"):      name,

diff --git a/controllers/goharbor/core/deployments.go b/controllers/goharbor/core/deployments.go
@@ -347,15 +347,6 @@ func (r *Reconciler) GetDeployment(ctx context.Context, core *goharborv1alpha2.C
 		envs = append(envs, adapterURL)
 	}
 
-	if core.Spec.Components.Trivy != nil {
-		adapterURLConfig, err := harbor.EnvVar(common.TrivyAdapterURL, harbor.Value(core.Spec.Components.Trivy.AdapterURL))
-		if err != nil {
-			return nil, errors.Wrap(err, "cannot configure trivy")
-		}
-
-		envs = append(envs, adapterURLConfig)
-	}
-
 	if core.Spec.Components.NotaryServer != nil {
 		envs = append(envs, corev1.EnvVar{
 			Name:  "NOTARY_URL",
@@ -448,9 +439,11 @@ func (r *Reconciler) GetDeployment(ctx context.Context, core *goharborv1alpha2.C
 						Ports: []corev1.ContainerPort{{
 							Name:          harbormetav1.CoreHTTPPortName,
 							ContainerPort: httpPort,
+							Protocol:      corev1.ProtocolTCP,
 						}, {
 							Name:          harbormetav1.CoreHTTPSPortName,
 							ContainerPort: httpsPort,
+							Protocol:      corev1.ProtocolTCP,
 						}},
 
 						// https://github.com/goharbor/harbor/blob/master/make/photon/prepare/templates/core/env.jinja

diff --git a/controllers/goharbor/core/services.go b/controllers/goharbor/core/services.go
@@ -24,10 +24,12 @@ func (r *Reconciler) GetService(ctx context.Context, core *goharborv1alpha2.Core
 				Name:       harbormetav1.CoreHTTPPortName,
 				Port:       harbormetav1.HTTPPort,
 				TargetPort: intstr.FromString(harbormetav1.CoreHTTPPortName),
+				Protocol:   corev1.ProtocolTCP,
 			}, {
 				Name:       harbormetav1.CoreHTTPSPortName,
 				Port:       harbormetav1.HTTPSPort,
 				TargetPort: intstr.FromString(harbormetav1.CoreHTTPSPortName),
+				Protocol:   corev1.ProtocolTCP,
 			}},
 			Selector: map[string]string{
 				r.Label("name"):      name,

diff --git a/controllers/goharbor/harbor/ingresses.go b/controllers/goharbor/harbor/ingresses.go
@@ -22,13 +22,13 @@ const (
 
 type CoreIngress graph.Resource
 
-func (r *Reconciler) AddCoreIngress(ctx context.Context, harbor *goharborv1alpha2.Harbor, core Core, portal Portal, registry Registry) (CoreIngress, error) {
+func (r *Reconciler) AddCoreIngress(ctx context.Context, harbor *goharborv1alpha2.Harbor, core Core, portal Portal) (CoreIngress, error) {
 	ingress, err := r.GetCoreIngress(ctx, harbor)
 	if err != nil {
 		return nil, errors.Wrap(err, "cannot get core ingress")
 	}
 
-	ingressRes, err := r.Controller.AddIngressToManage(ctx, ingress, core, portal, registry)
+	ingressRes, err := r.Controller.AddIngressToManage(ctx, ingress, core, portal)
 
 	return CoreIngress(ingressRes), errors.Wrap(err, "cannot add core ingress")
 }

diff --git a/controllers/goharbor/harbor/jobservice.go b/controllers/goharbor/harbor/jobservice.go
@@ -71,7 +71,7 @@ func (r *Reconciler) GetJobServiceSecret(ctx context.Context, harbor *goharborv1
 
 type JobService graph.Resource
 
-func (r *Reconciler) AddJobService(ctx context.Context, harbor *goharborv1alpha2.Harbor, certificate JobServiceInternalCertificate, core Core, coreSecret CoreSecret, jobServiceSecret JobServiceSecret) (JobService, error) {
+func (r *Reconciler) AddJobService(ctx context.Context, harbor *goharborv1alpha2.Harbor, core Core, certificate JobServiceInternalCertificate, coreSecret CoreSecret, jobServiceSecret JobServiceSecret) (JobService, error) {
 	jobservice, err := r.GetJobService(ctx, harbor)
 	if err != nil {
 		return nil, errors.Wrap(err, "get")

diff --git a/controllers/goharbor/harbor/notaryserver.go b/controllers/goharbor/harbor/notaryserver.go
@@ -102,6 +102,10 @@ func (r *Reconciler) GetNotaryServerCertificate(ctx context.Context, harbor *goh
 	secretName := r.NormalizeName(ctx, harbor.GetName(), controllers.NotaryServer.String(), "authentication")
 
 	return &certv1.Certificate{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       certv1.CertificateKind,
+			APIVersion: certv1.SchemeGroupVersion.String(),
+		},
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      r.NormalizeName(ctx, harbor.GetName(), controllers.NotaryServer.String(), "authentication"),
 			Namespace: harbor.GetNamespace(),

diff --git a/controllers/goharbor/harbor/resources.go b/controllers/goharbor/harbor/resources.go
@@ -105,7 +105,7 @@ func (r *Reconciler) AddResources(ctx context.Context, resource resources.Resour
 		return errors.Wrapf(err, "cannot add %s", controllers.Trivy)
 	}
 
-	_, err = r.AddCoreIngress(ctx, harbor, core, portal, registry)
+	_, err = r.AddCoreIngress(ctx, harbor, core, portal)
 	if err != nil {
 		return errors.Wrapf(err, "cannot add %s ingress", controllers.Core)
 	}

diff --git a/controllers/goharbor/jobservice/deployments.go b/controllers/goharbor/jobservice/deployments.go
@@ -250,9 +250,11 @@ func (r *Reconciler) GetDeployment(ctx context.Context, jobservice *goharborv1al
 						Ports: []corev1.ContainerPort{{
 							Name:          harbormetav1.JobServiceHTTPPortName,
 							ContainerPort: httpPort,
+							Protocol:      corev1.ProtocolTCP,
 						}, {
 							Name:          harbormetav1.JobServiceHTTPSPortName,
 							ContainerPort: httpsPort,
+							Protocol:      corev1.ProtocolTCP,
 						}},
 
 						// https://github.com/goharbor/harbor/blob/master/make/photon/prepare/templates/jobservice/env.jinja

diff --git a/controllers/goharbor/jobservice/services.go b/controllers/goharbor/jobservice/services.go
@@ -24,10 +24,12 @@ func (r *Reconciler) GetService(ctx context.Context, jobservice *goharborv1alpha
 				Name:       harbormetav1.JobServiceHTTPPortName,
 				Port:       harbormetav1.HTTPPort,
 				TargetPort: intstr.FromString(harbormetav1.JobServiceHTTPPortName),
+				Protocol:   corev1.ProtocolTCP,
 			}, {
 				Name:       harbormetav1.JobServiceHTTPSPortName,
 				Port:       harbormetav1.HTTPSPort,
 				TargetPort: intstr.FromString(harbormetav1.JobServiceHTTPSPortName),
+				Protocol:   corev1.ProtocolTCP,
 			}},
 			Selector: map[string]string{
 				r.Label("name"):      name,

diff --git a/controllers/goharbor/notaryserver/deployments.go b/controllers/goharbor/notaryserver/deployments.go
@@ -172,6 +172,7 @@ func (r *Reconciler) GetDeployment(ctx context.Context, notary *goharborv1alpha2
 						Ports: []corev1.ContainerPort{{
 							ContainerPort: apiPort,
 							Name:          harbormetav1.NotaryServerAPIPortName,
+							Protocol:      corev1.ProtocolTCP,
 						}},
 						VolumeMounts: volumeMounts,
 						Env:          migrationEnvs,

diff --git a/controllers/goharbor/notaryserver/resources.go b/controllers/goharbor/notaryserver/resources.go
@@ -4,6 +4,7 @@ import (
 	"context"
 
 	goharborv1alpha2 "github.com/goharbor/harbor-operator/apis/goharbor.io/v1alpha2"
+	harbormetav1 "github.com/goharbor/harbor-operator/apis/meta/v1alpha1"
 	serrors "github.com/goharbor/harbor-operator/pkg/controller/errors"
 	"github.com/goharbor/harbor-operator/pkg/graph"
 	"github.com/goharbor/harbor-operator/pkg/resources"
@@ -30,12 +31,12 @@ func (r *Reconciler) AddResources(ctx context.Context, resource resources.Resour
 	var storageSecret graph.Resource
 
 	if notaryserver.Spec.Storage.Postgres.PasswordRef != "" {
-		storageSecret, err = r.AddExternalResource(ctx, &corev1.Secret{
+		storageSecret, err = r.AddExternalTypedSecret(ctx, &corev1.Secret{
 			ObjectMeta: metav1.ObjectMeta{
 				Name:      notaryserver.Spec.Storage.Postgres.PasswordRef,
 				Namespace: notaryserver.GetNamespace(),
 			},
-		})
+		}, harbormetav1.SecretTypePostgresql)
 		if err != nil {
 			return errors.Wrap(err, "cannot add migration secret")
 		}

diff --git a/controllers/goharbor/notaryserver/services.go b/controllers/goharbor/notaryserver/services.go
@@ -28,6 +28,7 @@ func (r *Reconciler) GetService(ctx context.Context, notary *goharborv1alpha2.No
 				Name:       harbormetav1.NotaryServerAPIPortName,
 				Port:       notary.Spec.TLS.GetInternalPort(),
 				TargetPort: intstr.FromString(harbormetav1.NotaryServerAPIPortName),
+				Protocol:   corev1.ProtocolTCP,
 			}},
 			Selector: map[string]string{
 				r.Label("name"):      name,

diff --git a/controllers/goharbor/notarysigner/deployments.go b/controllers/goharbor/notarysigner/deployments.go
@@ -129,6 +129,7 @@ func (r *Reconciler) GetDeployment(ctx context.Context, notary *goharborv1alpha2
 						Ports: []corev1.ContainerPort{{
 							ContainerPort: goharborv1alpha2.NotarySignerAPIPort,
 							Name:          harbormetav1.NotarySignerAPIPortName,
+							Protocol:      corev1.ProtocolTCP,
 						}},
 						EnvFrom: []corev1.EnvFromSource{{
 							Prefix: "NOTARY_SIGNER_",

diff --git a/controllers/goharbor/notarysigner/resources.go b/controllers/goharbor/notarysigner/resources.go
@@ -4,6 +4,7 @@ import (
 	"context"
 
 	goharborv1alpha2 "github.com/goharbor/harbor-operator/apis/goharbor.io/v1alpha2"
+	harbormetav1 "github.com/goharbor/harbor-operator/apis/meta/v1alpha1"
 	serrors "github.com/goharbor/harbor-operator/pkg/controller/errors"
 	"github.com/goharbor/harbor-operator/pkg/graph"
 	"github.com/goharbor/harbor-operator/pkg/resources"
@@ -35,12 +36,12 @@ func (r *Reconciler) AddResources(ctx context.Context, resource resources.Resour
 	var storageSecret graph.Resource
 
 	if notary.Spec.Storage.Postgres.PasswordRef != "" {
-		storageSecret, err = r.AddExternalResource(ctx, &corev1.Secret{
+		storageSecret, err = r.AddExternalTypedSecret(ctx, &corev1.Secret{
 			ObjectMeta: metav1.ObjectMeta{
 				Name:      notary.Spec.Storage.Postgres.PasswordRef,
 				Namespace: notary.GetNamespace(),
 			},
-		})
+		}, harbormetav1.SecretTypePostgresql)
 		if err != nil {
 			return errors.Wrap(err, "cannot add migration secret")
 		}

diff --git a/controllers/goharbor/notarysigner/services.go b/controllers/goharbor/notarysigner/services.go
@@ -28,6 +28,7 @@ func (r *Reconciler) GetService(ctx context.Context, notary *goharborv1alpha2.No
 				Name:       harbormetav1.NotarySignerAPIPortName,
 				Port:       goharborv1alpha2.NotarySignerAPIPort,
 				TargetPort: intstr.FromString(harbormetav1.NotarySignerAPIPortName),
+				Protocol:   corev1.ProtocolTCP,
 			}},
 			Selector: map[string]string{
 				r.Label("name"):      name,

diff --git a/controllers/goharbor/portal/deployments.go b/controllers/goharbor/portal/deployments.go
@@ -134,9 +134,11 @@ func (r *Reconciler) GetDeployment(ctx context.Context, portal *goharborv1alpha2
 							Ports: []corev1.ContainerPort{{
 								Name:          harbormetav1.JobServiceHTTPPortName,
 								ContainerPort: httpPort,
+								Protocol:      corev1.ProtocolTCP,
 							}, {
 								Name:          harbormetav1.JobServiceHTTPSPortName,
 								ContainerPort: httpsPort,
+								Protocol:      corev1.ProtocolTCP,
 							}},
 							Env:          envs,
 							VolumeMounts: volumeMounts,

diff --git a/controllers/goharbor/portal/services.go b/controllers/goharbor/portal/services.go
@@ -24,10 +24,12 @@ func (r *Reconciler) GetService(ctx context.Context, portal *goharborv1alpha2.Po
 				Name:       harbormetav1.PortalHTTPPortName,
 				Port:       harbormetav1.HTTPPort,
 				TargetPort: intstr.FromString(harbormetav1.PortalHTTPPortName),
+				Protocol:   corev1.ProtocolTCP,
 			}, {
 				Name:       harbormetav1.PortalHTTPSPortName,
 				Port:       harbormetav1.HTTPSPort,
 				TargetPort: intstr.FromString(harbormetav1.PortalHTTPSPortName),
+				Protocol:   corev1.ProtocolTCP,
 			}},
 			Selector: map[string]string{
 				r.Label("name"):      name,

diff --git a/controllers/goharbor/registry/deployments.go b/controllers/goharbor/registry/deployments.go
@@ -248,9 +248,11 @@ func (r *Reconciler) GetDeployment(ctx context.Context, registry *goharborv1alph
 						Ports: []corev1.ContainerPort{{
 							ContainerPort: apiPort,
 							Name:          harbormetav1.RegistryAPIPortName,
+							Protocol:      corev1.ProtocolTCP,
 						}, {
 							ContainerPort: metricsPort,
 							Name:          harbormetav1.RegistryMetricsPortName,
+							Protocol:      corev1.ProtocolTCP,
 						}},
 						LivenessProbe: &corev1.Probe{
 							Handler: corev1.Handler{

diff --git a/controllers/goharbor/registry/services.go b/controllers/goharbor/registry/services.go
@@ -24,10 +24,12 @@ func (r *Reconciler) GetService(ctx context.Context, registry *goharborv1alpha2.
 				Name:       harbormetav1.RegistryAPIPortName,
 				Port:       registry.Spec.HTTP.TLS.GetInternalPort(),
 				TargetPort: intstr.FromString(harbormetav1.RegistryAPIPortName),
+				Protocol:   corev1.ProtocolTCP,
 			}, {
 				Name:       harbormetav1.RegistryMetricsPortName,
 				Port:       registry.Spec.HTTP.TLS.GetInternalPort() + 1,
 				TargetPort: intstr.FromString(harbormetav1.RegistryMetricsPortName),
+				Protocol:   corev1.ProtocolTCP,
 			}},
 			Selector: map[string]string{
 				r.Label("name"):      name,

diff --git a/controllers/goharbor/registryctl/deployments.go b/controllers/goharbor/registryctl/deployments.go
@@ -165,9 +165,11 @@ func (r *Reconciler) GetDeployment(ctx context.Context, registryCtl *goharborv1a
 	registryContainer.Ports = []corev1.ContainerPort{{
 		Name:          harbormetav1.RegistryControllerHTTPPortName,
 		ContainerPort: httpPort,
+		Protocol:      corev1.ProtocolTCP,
 	}, {
 		Name:          harbormetav1.RegistryControllerHTTPSPortName,
 		ContainerPort: httpsPort,
+		Protocol:      corev1.ProtocolTCP,
 	}}
 
 	port := harbormetav1.RegistryControllerHTTPPortName

diff --git a/controllers/goharbor/registryctl/services.go b/controllers/goharbor/registryctl/services.go
@@ -28,10 +28,12 @@ func (r *Reconciler) GetService(ctx context.Context, registryCtl *goharborv1alph
 				Name:       harbormetav1.RegistryControllerHTTPPortName,
 				Port:       harbormetav1.HTTPPort,
 				TargetPort: intstr.FromString(harbormetav1.RegistryControllerHTTPPortName),
+				Protocol:   corev1.ProtocolTCP,
 			}, {
 				Name:       harbormetav1.RegistryControllerHTTPSPortName,
 				Port:       harbormetav1.HTTPSPort,
 				TargetPort: intstr.FromString(harbormetav1.RegistryControllerHTTPSPortName),
+				Protocol:   corev1.ProtocolTCP,
 			}},
 			Selector: map[string]string{
 				r.Label("name"):      name,

diff --git a/controllers/goharbor/trivy/deployments.go b/controllers/goharbor/trivy/deployments.go
@@ -204,9 +204,11 @@ func (r *Reconciler) GetDeployment(ctx context.Context, trivy *goharborv1alpha2.
 						Ports: []corev1.ContainerPort{{
 							Name:          harbormetav1.TrivyHTTPPortName,
 							ContainerPort: httpPort,
+							Protocol:      corev1.ProtocolTCP,
 						}, {
 							Name:          harbormetav1.TrivyHTTPSPortName,
 							ContainerPort: httpsPort,
+							Protocol:      corev1.ProtocolTCP,
 						}},
 
 						Env:          envs,

diff --git a/controllers/goharbor/trivy/services.go b/controllers/goharbor/trivy/services.go
@@ -39,10 +39,12 @@ func (r *Reconciler) GetService(ctx context.Context, trivy *goharborv1alpha2.Tri
 				Name:       harbormetav1.TrivyHTTPPortName,
 				Port:       harbormetav1.HTTPPort,
 				TargetPort: intstr.FromString(harbormetav1.TrivyHTTPPortName),
+				Protocol:   corev1.ProtocolTCP,
 			}, {
 				Name:       harbormetav1.TrivyHTTPSPortName,
 				Port:       harbormetav1.HTTPSPort,
 				TargetPort: intstr.FromString(harbormetav1.TrivyHTTPSPortName),
+				Protocol:   corev1.ProtocolTCP,
 			}},
 			Selector: map[string]string{
 				r.Label("name"):      name,