Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Pruning improvements #818

Merged
merged 3 commits into from
Aug 20, 2021
Merged

Pruning improvements #818

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
6c6db41
MAISTRA-2607 Add option to log API requests performed by the operator
luksa Aug 20, 2021
14095ff
MAISTRA-1621 Ensure gateways in non-cp namespaces are pruned
luksa Aug 3, 2021
2340a7a
MAISTRA-2608 Improve pruner performance by eliminating repeated API d…
luksa Aug 5, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 24 additions & 0 deletions cmd/manager/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"errors"
"flag"
"fmt"
"net/http"
"os"
"strings"

Expand Down Expand Up @@ -68,6 +69,9 @@ func main() {
pflag.String("defaultTemplatesDir", "", "The root location of the default templates.")
pflag.String("userTemplatesDir", "", "The root location of the user supplied templates.")

var logAPIRequests bool
pflag.BoolVar(&logAPIRequests, "logAPIRequests", false, "Log API requests performed by the operator.")

// config file
configFile := ""
pflag.StringVar(&configFile, "config", "/etc/istio-operator/config.properties", "The root location of the user supplied templates.")
Expand Down Expand Up @@ -110,6 +114,14 @@ func main() {
cfg.Burst = common.Config.Controller.APIBurst
cfg.QPS = common.Config.Controller.APIQPS

if logAPIRequests {
cfg.Wrap(func(rt http.RoundTripper) http.RoundTripper {
return requestLogger{
rt: rt,
}
})
}

ctx := context.Background()
// Become the leader before proceeding
err = leader.Become(ctx, "istio-operator-lock")
Expand Down Expand Up @@ -304,3 +316,15 @@ func patchProperties(file string) (map[string]interface{}, error) {
}
return retVal, nil
}

type requestLogger struct {
rt http.RoundTripper
}

func (rl requestLogger) RoundTrip(req *http.Request) (*http.Response, error) {
log := common.LogFromContext(req.Context())
log.Info("Performing API request", "method", req.Method, "URL", req.URL)
return rl.rt.RoundTrip(req)
}

var _ http.RoundTripper = requestLogger{}
2 changes: 1 addition & 1 deletion pkg/controller/common/context.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ type ContextValueKey string
var (
logContextKey ContextValueKey = "github.com/maistra/istio-operator/pkg/controller/common/logr.Logger"

fallBackLogger = logf.Log.WithName("FALLBACK-LOGGER")
fallBackLogger = logf.Log
)

// NewContext creates a new context without an associated Logger
Expand Down
43 changes: 43 additions & 0 deletions pkg/controller/servicemesh/controlplane/common_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import (
"go.uber.org/zap/zapcore"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
v1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
Expand All @@ -34,6 +35,46 @@ import (

var ctx = common.NewContextWithLog(context.Background(), logf.Log)

var kialiCRD = v1.CustomResourceDefinition{
ObjectMeta: metav1.ObjectMeta{Name: "kialis.kiali.io"},
Spec: v1.CustomResourceDefinitionSpec{
Group: "kiali.io",
Names: v1.CustomResourceDefinitionNames{
Plural: "kialis",
Singular: "kiali",
Kind: "Kiali",
ListKind: "KialiList",
},
Scope: "Namespaced",
Versions: []v1.CustomResourceDefinitionVersion{
{
Name: "v1alpha1",
Served: true,
},
},
},
}

var jaegerCRD = v1.CustomResourceDefinition{
ObjectMeta: metav1.ObjectMeta{Name: "jaegers.jaegertracing.io"},
Spec: v1.CustomResourceDefinitionSpec{
Group: "jaegertracing.io",
Names: v1.CustomResourceDefinitionNames{
Plural: "jaegers",
Singular: "jaeger",
Kind: "Jaeger",
ListKind: "JaegerList",
},
Scope: "Namespaced",
Versions: []v1.CustomResourceDefinitionVersion{
{
Name: "v1",
Served: true,
},
},
},
}

func init() {
hacks.CacheSyncWaitDuration = 0
}
Expand Down Expand Up @@ -70,6 +111,8 @@ func RunSimpleInstallTest(t *testing.T, testCases []IntegrationTestCase) {
Resources: []runtime.Object{
&corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: controlPlaneNamespace}},
&corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: operatorNamespace}},
&kialiCRD,
&jaegerCRD,
},
GroupResources: []*restmapper.APIGroupResources{
CNIGroupResources,
Expand Down
192 changes: 111 additions & 81 deletions pkg/controller/servicemesh/controlplane/pruner.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,116 +4,140 @@ import (
"context"
"fmt"

"github.com/maistra/istio-operator/pkg/controller/common"
v1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
"k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/meta"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/selection"

"github.com/maistra/istio-operator/pkg/controller/common"

metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/selection"
utilerrors "k8s.io/apimachinery/pkg/util/errors"

"sigs.k8s.io/controller-runtime/pkg/client"
)

type pruneConfig struct {
gvk schema.GroupVersionKind
supportsDeleteCollection bool
}

var (
// XXX: move this into a ConfigMap so users can override things if they add new types in customized charts
// ordered by which types should be deleted, first to last
namespacedResources = map[schema.GroupVersionKind]pruneConfig{
gvk("kiali.io", "v1alpha1", "Kiali"): {supportsDeleteCollection: true},
gvk("autoscaling", "v2beta1", "HorizontalPodAutoscaler"): {supportsDeleteCollection: true},
gvk("policy", "v1beta1", "PodDisruptionBudget"): {supportsDeleteCollection: true},
gvk("route.openshift.io", "v1", "Route"): {supportsDeleteCollection: true},
gvk("apps", "v1", "Deployment"): {supportsDeleteCollection: true},
gvk("apps", "v1", "DaemonSet"): {supportsDeleteCollection: true},
gvk("apps", "v1", "StatefulSet"): {supportsDeleteCollection: true},
gvk("extensions", "v1beta1", "Ingress"): {supportsDeleteCollection: true},
gvk("", "v1", "Service"): {supportsDeleteCollection: false},
gvk("", "v1", "Endpoints"): {supportsDeleteCollection: true},
gvk("", "v1", "ConfigMap"): {supportsDeleteCollection: true},
gvk("", "v1", "PersistentVolumeClaim"): {supportsDeleteCollection: true},
gvk("", "v1", "Pod"): {supportsDeleteCollection: true},
gvk("", "v1", "Secret"): {supportsDeleteCollection: true},
gvk("", "v1", "ServiceAccount"): {supportsDeleteCollection: true},
gvk("networking.k8s.io", "v1", "NetworkPolicy"): {supportsDeleteCollection: true},
gvk("rbac.authorization.k8s.io", "v1beta1", "RoleBinding"): {supportsDeleteCollection: true},
gvk("rbac.authorization.k8s.io", "v1", "RoleBinding"): {supportsDeleteCollection: true},
gvk("rbac.authorization.k8s.io", "v1beta1", "Role"): {supportsDeleteCollection: true},
gvk("rbac.authorization.k8s.io", "v1", "Role"): {supportsDeleteCollection: true},
gvk("authentication.istio.io", "v1alpha1", "Policy"): {supportsDeleteCollection: true},
gvk("config.istio.io", "v1alpha2", "adapter"): {supportsDeleteCollection: true},
gvk("config.istio.io", "v1alpha2", "attributemanifest"): {supportsDeleteCollection: true},
gvk("config.istio.io", "v1alpha2", "handler"): {supportsDeleteCollection: true},
gvk("config.istio.io", "v1alpha2", "instance"): {supportsDeleteCollection: true},
gvk("config.istio.io", "v1alpha2", "kubernetes"): {supportsDeleteCollection: true},
gvk("config.istio.io", "v1alpha2", "logentry"): {supportsDeleteCollection: true},
gvk("config.istio.io", "v1alpha2", "metric"): {supportsDeleteCollection: true},
gvk("config.istio.io", "v1alpha2", "rule"): {supportsDeleteCollection: true},
gvk("config.istio.io", "v1alpha2", "template"): {supportsDeleteCollection: true},
gvk("networking.istio.io", "v1alpha3", "DestinationRule"): {supportsDeleteCollection: true},
gvk("networking.istio.io", "v1beta1", "DestinationRule"): {supportsDeleteCollection: true},
gvk("networking.istio.io", "v1alpha3", "EnvoyFilter"): {supportsDeleteCollection: true},
gvk("networking.istio.io", "v1alpha3", "Gateway"): {supportsDeleteCollection: true},
gvk("networking.istio.io", "v1beta1", "Gateway"): {supportsDeleteCollection: true},
gvk("networking.istio.io", "v1alpha3", "ServiceEntry"): {supportsDeleteCollection: true},
gvk("networking.istio.io", "v1beta1", "ServiceEntry"): {supportsDeleteCollection: true},
gvk("networking.istio.io", "v1alpha3", "Sidecar"): {supportsDeleteCollection: true},
gvk("networking.istio.io", "v1beta1", "Sidecar"): {supportsDeleteCollection: true},
gvk("networking.istio.io", "v1alpha3", "VirtualService"): {supportsDeleteCollection: true},
gvk("networking.istio.io", "v1beta1", "VirtualService"): {supportsDeleteCollection: true},
gvk("jaegertracing.io", "v1", "Jaeger"): {supportsDeleteCollection: true},
gvk("authentication.maistra.io", "v1", "ServiceMeshPolicy"): {supportsDeleteCollection: true},
gvk("security.istio.io", "v1beta1", "AuthorizationPolicy"): {supportsDeleteCollection: true},
gvk("security.istio.io", "v1beta1", "PeerAuthentication"): {supportsDeleteCollection: true},
gvk("security.istio.io", "v1beta1", "RequestAuthentication"): {supportsDeleteCollection: true},
gvk("networking.istio.io", "v1alpha3", "WorkloadEntry"): {supportsDeleteCollection: true},
}

// ordered by which types should be deleted, first to last
nonNamespacedResources = map[schema.GroupVersionKind]pruneConfig{
gvk("admissionregistration.k8s.io", "v1beta1", "MutatingWebhookConfiguration"): {supportsDeleteCollection: true},
gvk("admissionregistration.k8s.io", "v1beta1", "ValidatingWebhookConfiguration"): {supportsDeleteCollection: true},
gvk("certmanager.k8s.io", "v1alpha1", "ClusterIssuer"): {supportsDeleteCollection: true},
gvk("rbac.authorization.k8s.io", "v1", "ClusterRole"): {supportsDeleteCollection: true},
gvk("rbac.authorization.k8s.io", "v1", "ClusterRoleBinding"): {supportsDeleteCollection: true},
builtinTypes = []schema.GroupVersionKind{
gvk("autoscaling", "v2beta1", "HorizontalPodAutoscaler"),
gvk("policy", "v1beta1", "PodDisruptionBudget"),
gvk("route.openshift.io", "v1", "Route"),
gvk("apps", "v1", "Deployment"),
gvk("apps", "v1", "DaemonSet"),
gvk("apps", "v1", "StatefulSet"),
gvk("extensions", "v1beta1", "Ingress"),
gvk("", "v1", "Service"),
gvk("", "v1", "Endpoints"),
gvk("", "v1", "ConfigMap"),
gvk("", "v1", "PersistentVolumeClaim"),
gvk("", "v1", "Pod"),
gvk("", "v1", "Secret"),
gvk("", "v1", "ServiceAccount"),
gvk("networking.k8s.io", "v1", "NetworkPolicy"),
gvk("rbac.authorization.k8s.io", "v1", "RoleBinding"),
gvk("rbac.authorization.k8s.io", "v1", "Role"),
gvk("admissionregistration.k8s.io", "v1beta1", "MutatingWebhookConfiguration"),
gvk("admissionregistration.k8s.io", "v1beta1", "ValidatingWebhookConfiguration"),
gvk("rbac.authorization.k8s.io", "v1", "ClusterRole"),
gvk("rbac.authorization.k8s.io", "v1", "ClusterRoleBinding"),
}

crds = map[schema.GroupKind]struct{}{
gk("kiali.io", "Kiali"): {},
gk("jaegertracing.io", "Jaeger"): {},
gk("config.istio.io", "adapter"): {},
gk("config.istio.io", "attributemanifest"): {},
gk("config.istio.io", "handler"): {},
gk("config.istio.io", "instance"): {},
gk("config.istio.io", "kubernetes"): {},
gk("config.istio.io", "logentry"): {},
gk("config.istio.io", "metric"): {},
gk("config.istio.io", "rule"): {},
gk("config.istio.io", "template"): {},
gk("networking.istio.io", "DestinationRule"): {},
gk("networking.istio.io", "EnvoyFilter"): {},
gk("networking.istio.io", "Gateway"): {},
gk("networking.istio.io", "ServiceEntry"): {},
gk("networking.istio.io", "Sidecar"): {},
gk("networking.istio.io", "VirtualService"): {},
gk("networking.istio.io", "WorkloadEntry"): {},
gk("authentication.istio.io", "Policy"): {},
gk("authentication.maistra.io", "ServiceMeshPolicy"): {},
gk("security.istio.io", "AuthorizationPolicy"): {},
gk("security.istio.io", "PeerAuthentication"): {},
gk("security.istio.io", "RequestAuthentication"): {},
gk("certmanager.k8s.io", "ClusterIssuer"): {},
}
)

func (r *controlPlaneInstanceReconciler) prune(ctx context.Context, generation string) error {
allErrors := []error{}
err := r.pruneResources(ctx, namespacedResources, generation, r.Instance.Namespace)
resourcesToPrune, err := r.findResourcesToPrune(ctx)
if err != nil {
allErrors = append(allErrors, err)
return err
}
err = r.pruneResources(ctx, namespacedResources, generation, r.OperatorNamespace)
if err != nil {
allErrors = append(allErrors, err)
return r.pruneResources(ctx, resourcesToPrune, generation)
}

func (r *controlPlaneInstanceReconciler) findResourcesToPrune(ctx context.Context) ([]pruneConfig, error) {
resourcesToPrune := []pruneConfig{}
for _, gvk := range builtinTypes {
resourcesToPrune = append(resourcesToPrune, pruneConfig{
gvk: gvk,
supportsDeleteCollection: false,
})
}
err = r.pruneResources(ctx, nonNamespacedResources, generation, "")

crdList := &v1.CustomResourceDefinitionList{}
err := r.Client.List(ctx, crdList)
if err != nil {
allErrors = append(allErrors, err)
return nil, err
}
return utilerrors.NewAggregate(allErrors)
for _, crd := range crdList.Items {
if _, exists := crds[gk(crd.Spec.Group, crd.Spec.Names.Kind)]; exists {
version := getVersion(crd)
if version == "" {
continue
}
resourcesToPrune = append(resourcesToPrune, pruneConfig{
gvk: schema.GroupVersionKind{
Group: crd.Spec.Group,
Version: version,
Kind: crd.Spec.Names.Kind,
},
supportsDeleteCollection: true,
})
}
}
return resourcesToPrune, nil
}

func (r *controlPlaneInstanceReconciler) pruneResources(ctx context.Context, gvks map[schema.GroupVersionKind]pruneConfig, instanceGeneration string, namespace string) error {
func getVersion(crd v1.CustomResourceDefinition) string {
for _, version := range crd.Spec.Versions {
if version.Served {
return version.Name
}
}
return ""
}

func (r *controlPlaneInstanceReconciler) pruneResources(ctx context.Context, pruneConfigs []pruneConfig, instanceGeneration string) error {
log := common.LogFromContext(ctx)

allErrors := []error{}
for gvk, pruneConfig := range gvks {
for _, pruneConfig := range pruneConfigs {
gvk := pruneConfig.gvk
log.Info("pruning resources", "type", gvk.String())
var err error
if pruneConfig.supportsDeleteCollection {
err = r.pruneAll(ctx, gvk, instanceGeneration, namespace)
err = r.pruneAll(ctx, gvk, instanceGeneration)
} else {
err = r.pruneIndividually(ctx, gvk, instanceGeneration, namespace)
err = r.pruneIndividually(ctx, gvk, instanceGeneration)
}
if err != nil {
log.Error(err, "Error pruning resources", "type", gvk.String())
Expand All @@ -123,14 +147,14 @@ func (r *controlPlaneInstanceReconciler) pruneResources(ctx context.Context, gvk
return utilerrors.NewAggregate(allErrors)
}

func (r *controlPlaneInstanceReconciler) pruneIndividually(ctx context.Context, gvk schema.GroupVersionKind, instanceGeneration string, namespace string) error {
func (r *controlPlaneInstanceReconciler) pruneIndividually(ctx context.Context, gvk schema.GroupVersionKind, instanceGeneration string) error {
labelSelector, err := createLabelSelector(r.Instance.Namespace, instanceGeneration)
if err != nil {
return err
}
objects := &unstructured.UnstructuredList{}
objects.SetGroupVersionKind(gvk)
err = r.Client.List(ctx, objects, client.InNamespace(namespace), client.MatchingLabelsSelector{Selector: labelSelector})
err = r.Client.List(ctx, objects, client.MatchingLabelsSelector{Selector: labelSelector})
if err != nil {
if meta.IsNoMatchError(err) || errors.IsNotFound(err) {
return nil
Expand All @@ -146,7 +170,7 @@ func (r *controlPlaneInstanceReconciler) pruneIndividually(ctx context.Context,
return nil
}

func (r *controlPlaneInstanceReconciler) pruneAll(ctx context.Context, gvk schema.GroupVersionKind, instanceGeneration string, namespace string) error {
func (r *controlPlaneInstanceReconciler) pruneAll(ctx context.Context, gvk schema.GroupVersionKind, instanceGeneration string) error {
labelSelector, err := createLabelSelector(r.Instance.Namespace, instanceGeneration)
if err != nil {
return err
Expand All @@ -156,7 +180,6 @@ func (r *controlPlaneInstanceReconciler) pruneAll(ctx context.Context, gvk schem
object.SetGroupVersionKind(gvk)
err = r.Client.DeleteAllOf(ctx,
object,
client.InNamespace(namespace),
client.MatchingLabelsSelector{Selector: labelSelector},
client.PropagationPolicy(metav1.DeletePropagationBackground))

Expand All @@ -174,6 +197,13 @@ func gvk(group, version, kind string) schema.GroupVersionKind {
}
}

func gk(group, kind string) schema.GroupKind {
return schema.GroupKind{
Group: group,
Kind: kind,
}
}

func createLabelSelector(meshNamespace, meshGeneration string) (labels.Selector, error) {
ownerRequirement, err := labels.NewRequirement(common.OwnerKey, selection.Equals, []string{meshNamespace})
if err != nil {
Expand Down
Loading