This repo contains a basic Helm operator with its chart that aim to reproduce issue operator-framework/operator-sdk#6220. The operator was created using the command operator-sdk init.
The chart contains 10 Ingresses, 10 Services, 10 Deployments and 10 ConfigMaps. The redondency here is to trigger the issue, which is somewhat random.
- Start a K8s cluster and put it in kubectl context
- Build and deploy the operator using
makeandmake deploy - Create 10 different namespaces, and in each of those 10 CRs from the sample template example command:
for i in {1..10}; do
echo $i
kubectl create ns namespace-$i
kubectl apply -n namespace-$i -f config/samples/agi_v1_apigroupissue.yaml
done
- Check the logs to see the error
These are logs from my own reproduction on Minikube. After a while, all reconciliations are successful:
{"level":"info","ts":1672653128.0212498,"logger":"cmd","msg":"Version","Go Version":"go1.19.4","GOOS":"linux","GOARCH":"amd64","helm-operator":"v1.26.0","commit":"cbeec475e4612e19f1047ff7014342afe93f60d2"}
{"level":"info","ts":1672653128.0254426,"logger":"cmd","msg":"Watch namespaces not configured by environment variable WATCH_NAMESPACE or file. Watching all namespaces.","Namespace":""}
{"level":"info","ts":1672653128.0444975,"logger":"controller-runtime.metrics","msg":"Metrics server is starting to listen","addr":"127.0.0.1:8080"}
{"level":"info","ts":1672653128.045502,"logger":"helm.controller","msg":"Watching resource","apiVersion":"agi.symphony.com/v1","kind":"ApiGroupIssue","namespace":"","reconcilePeriod":"1m0s"}
{"level":"info","ts":1672653128.045884,"msg":"Starting server","path":"/metrics","kind":"metrics","addr":"127.0.0.1:8080"}
{"level":"info","ts":1672653128.0460963,"msg":"Starting server","kind":"health probe","addr":"[::]:8081"}
I0102 09:52:08.046280 1 leaderelection.go:248] attempting to acquire leader lease api-group-issue-helm-op-system/api-group-issue-helm-op...
I0102 09:52:34.605973 1 leaderelection.go:258] successfully acquired lease api-group-issue-helm-op-system/api-group-issue-helm-op
{"level":"info","ts":1672653154.6103048,"msg":"Starting EventSource","controller":"apigroupissue-controller","source":"kind source: *unstructured.Unstructured"}
{"level":"info","ts":1672653154.6103716,"msg":"Starting Controller","controller":"apigroupissue-controller"}
{"level":"info","ts":1672653154.7121198,"msg":"Starting workers","controller":"apigroupissue-controller","worker count":2}
{"level":"info","ts":1672653155.74566,"msg":"Starting EventSource","controller":"apigroupissue-controller","source":"kind source: *unstructured.Unstructured"}
{"level":"info","ts":1672653155.7457242,"logger":"helm.controller","msg":"Watching dependent resource","ownerApiVersion":"agi.symphony.com/v1","ownerKind":"ApiGroupIssue","apiVersion":"networking.k8s.io/v1","kind":"Ingress"}
{"level":"info","ts":1672653155.7458816,"msg":"Starting EventSource","controller":"apigroupissue-controller","source":"kind source: *unstructured.Unstructured"}
{"level":"info","ts":1672653155.7463424,"logger":"helm.controller","msg":"Watching dependent resource","ownerApiVersion":"agi.symphony.com/v1","ownerKind":"ApiGroupIssue","apiVersion":"v1","kind":"ConfigMap"}
{"level":"info","ts":1672653155.7469988,"msg":"Starting EventSource","controller":"apigroupissue-controller","source":"kind source: *unstructured.Unstructured"}
{"level":"info","ts":1672653155.7470498,"logger":"helm.controller","msg":"Watching dependent resource","ownerApiVersion":"agi.symphony.com/v1","ownerKind":"ApiGroupIssue","apiVersion":"v1","kind":"Service"}
{"level":"info","ts":1672653155.747249,"msg":"Starting EventSource","controller":"apigroupissue-controller","source":"kind source: *unstructured.Unstructured"}
{"level":"info","ts":1672653155.7472966,"logger":"helm.controller","msg":"Watching dependent resource","ownerApiVersion":"agi.symphony.com/v1","ownerKind":"ApiGroupIssue","apiVersion":"apps/v1","kind":"Deployment"}
{"level":"info","ts":1672653155.7688503,"logger":"helm.controller","msg":"Upgraded release","namespace":"tenant-1","name":"apigroupissue-sample","apiVersion":"agi.symphony.com/v1","kind":"ApiGroupIssue","release":"apigroupissue-sample","force":false}
{"level":"info","ts":1672653156.3067253,"logger":"helm.controller","msg":"Reconciled release","namespace":"tenant-1","name":"apigroupissue-sample","apiVersion":"agi.symphony.com/v1","kind":"ApiGroupIssue","release":"apigroupissue-sample"}
{"level":"info","ts":1672653216.3063955,"logger":"helm.controller","msg":"Reconciled release","namespace":"tenant-1","name":"apigroupissue-sample","apiVersion":"agi.symphony.com/v1","kind":"ApiGroupIssue","release":"apigroupissue-sample"}
{"level":"info","ts":1672653276.768517,"logger":"helm.controller","msg":"Reconciled release","namespace":"tenant-1","name":"apigroupissue-sample","apiVersion":"agi.symphony.com/v1","kind":"ApiGroupIssue","release":"apigroupissue-sample"}
{"level":"info","ts":1672653328.3830419,"logger":"helm.controller","msg":"Installed release","namespace":"tenant-2","name":"apigroupissue-sample","apiVersion":"agi.symphony.com/v1","kind":"ApiGroupIssue","release":"apigroupissue-sample"}
{"level":"info","ts":1672653329.7454607,"logger":"helm.controller","msg":"Installed release","namespace":"tenant-3","name":"apigroupissue-sample","apiVersion":"agi.symphony.com/v1","kind":"ApiGroupIssue","release":"apigroupissue-sample"}
{"level":"info","ts":1672653331.3118417,"logger":"helm.controller","msg":"Installed release","namespace":"tenant-4","name":"apigroupissue-sample","apiVersion":"agi.symphony.com/v1","kind":"ApiGroupIssue","release":"apigroupissue-sample"}
{"level":"info","ts":1672653331.9359,"logger":"helm.controller","msg":"Installed release","namespace":"tenant-5","name":"apigroupissue-sample","apiVersion":"agi.symphony.com/v1","kind":"ApiGroupIssue","release":"apigroupissue-sample"}
{"level":"info","ts":1672653333.5817177,"logger":"helm.controller","msg":"Installed release","namespace":"tenant-6","name":"apigroupissue-sample","apiVersion":"agi.symphony.com/v1","kind":"ApiGroupIssue","release":"apigroupissue-sample"}
{"level":"info","ts":1672653334.251942,"logger":"helm.controller","msg":"Installed release","namespace":"tenant-7","name":"apigroupissue-sample","apiVersion":"agi.symphony.com/v1","kind":"ApiGroupIssue","release":"apigroupissue-sample"}
{"level":"error","ts":1672653334.9715686,"logger":"helm.controller","msg":"Failed to reconcile release","namespace":"tenant-2","name":"apigroupissue-sample","apiVersion":"agi.symphony.com/v1","kind":"ApiGroupIssue","release":"apigroupissue-sample","error":"could not get object: configmaps.apps \"tenant-2-api-group-issue-cm-4\" is forbidden: User \"system:serviceaccount:api-group-issue-helm-op-system:api-group-issue-helm-op-controller-manager\" cannot get resource \"configmaps\" in API group \"apps\" in the namespace \"tenant-2\"","stacktrace":"github.com/operator-framework/operator-sdk/internal/helm/controller.HelmOperatorReconciler.Reconcile\n\t/workspace/internal/helm/controller/reconcile.go:363\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.13.0/pkg/internal/controller/controller.go:121\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.13.0/pkg/internal/controller/controller.go:320\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.13.0/pkg/internal/controller/controller.go:273\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.13.0/pkg/internal/controller/controller.go:234"}
{"level":"error","ts":1672653334.9843082,"msg":"Reconciler error","controller":"apigroupissue-controller","object":{"name":"apigroupissue-sample","namespace":"tenant-2"},"namespace":"tenant-2","name":"apigroupissue-sample","reconcileID":"205b8043-4808-449f-b819-381bc07c6b7a","error":"could not get object: configmaps.apps \"tenant-2-api-group-issue-cm-4\" is forbidden: User \"system:serviceaccount:api-group-issue-helm-op-system:api-group-issue-helm-op-controller-manager\" cannot get resource \"configmaps\" in API group \"apps\" in the namespace \"tenant-2\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.13.0/pkg/internal/controller/controller.go:326\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.13.0/pkg/internal/controller/controller.go:273\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.13.0/pkg/internal/controller/controller.go:234"}
{"level":"info","ts":1672653336.2019613,"logger":"helm.controller","msg":"Installed release","namespace":"tenant-8","name":"apigroupissue-sample","apiVersion":"agi.symphony.com/v1","kind":"ApiGroupIssue","release":"apigroupissue-sample"}
{"level":"info","ts":1672653337.0668228,"logger":"helm.controller","msg":"Installed release","namespace":"tenant-9","name":"apigroupissue-sample","apiVersion":"agi.symphony.com/v1","kind":"ApiGroupIssue","release":"apigroupissue-sample"}
{"level":"info","ts":1672653338.4566004,"logger":"helm.controller","msg":"Installed release","namespace":"tenant-10","name":"apigroupissue-sample","apiVersion":"agi.symphony.com/v1","kind":"ApiGroupIssue","release":"apigroupissue-sample"}
{"level":"info","ts":1672653339.3032305,"logger":"helm.controller","msg":"Reconciled release","namespace":"tenant-3","name":"apigroupissue-sample","apiVersion":"agi.symphony.com/v1","kind":"ApiGroupIssue","release":"apigroupissue-sample"}
{"level":"info","ts":1672653339.560869,"logger":"helm.controller","msg":"Reconciled release","namespace":"tenant-4","name":"apigroupissue-sample","apiVersion":"agi.symphony.com/v1","kind":"ApiGroupIssue","release":"apigroupissue-sample"}
{"level":"info","ts":1672653340.683022,"logger":"helm.controller","msg":"Reconciled release","namespace":"tenant-5","name":"apigroupissue-sample","apiVersion":"agi.symphony.com/v1","kind":"ApiGroupIssue","release":"apigroupissue-sample"}
{"level":"info","ts":1672653341.1962233,"logger":"helm.controller","msg":"Reconciled release","namespace":"tenant-6","name":"apigroupissue-sample","apiVersion":"agi.symphony.com/v1","kind":"ApiGroupIssue","release":"apigroupissue-sample"}
{"level":"error","ts":1672653342.256705,"logger":"helm.controller","msg":"Failed to reconcile release","namespace":"tenant-7","name":"apigroupissue-sample","apiVersion":"agi.symphony.com/v1","kind":"ApiGroupIssue","release":"apigroupissue-sample","error":"could not get object: ingresses \"tenant-7-api-group-issue-ing-8\" is forbidden: User \"system:serviceaccount:api-group-issue-helm-op-system:api-group-issue-helm-op-controller-manager\" cannot get resource \"ingresses\" in API group \"\" in the namespace \"tenant-7\"","stacktrace":"github.com/operator-framework/operator-sdk/internal/helm/controller.HelmOperatorReconciler.Reconcile\n\t/workspace/internal/helm/controller/reconcile.go:363\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.13.0/pkg/internal/controller/controller.go:121\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.13.0/pkg/internal/controller/controller.go:320\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.13.0/pkg/internal/controller/controller.go:273\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.13.0/pkg/internal/controller/controller.go:234"}
{"level":"error","ts":1672653342.2818904,"msg":"Reconciler error","controller":"apigroupissue-controller","object":{"name":"apigroupissue-sample","namespace":"tenant-7"},"namespace":"tenant-7","name":"apigroupissue-sample","reconcileID":"e4fe1f59-97d5-4350-b378-30d191a1e441","error":"could not get object: ingresses \"tenant-7-api-group-issue-ing-8\" is forbidden: User \"system:serviceaccount:api-group-issue-helm-op-system:api-group-issue-helm-op-controller-manager\" cannot get resource \"ingresses\" in API group \"\" in the namespace \"tenant-7\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.13.0/pkg/internal/controller/controller.go:326\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.13.0/pkg/internal/controller/controller.go:273\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.13.0/pkg/internal/controller/controller.go:234"}
{"level":"info","ts":1672653342.5295217,"logger":"helm.controller","msg":"Reconciled release","namespace":"tenant-2","name":"apigroupissue-sample","apiVersion":"agi.symphony.com/v1","kind":"ApiGroupIssue","release":"apigroupissue-sample"}
{"level":"info","ts":1672653344.385686,"logger":"helm.controller","msg":"Reconciled release","namespace":"tenant-8","name":"apigroupissue-sample","apiVersion":"agi.symphony.com/v1","kind":"ApiGroupIssue","release":"apigroupissue-sample"}
{"level":"info","ts":1672653344.4835887,"logger":"helm.controller","msg":"Reconciled release","namespace":"tenant-1","name":"apigroupissue-sample","apiVersion":"agi.symphony.com/v1","kind":"ApiGroupIssue","release":"apigroupissue-sample"}
{"level":"info","ts":1672653346.204334,"logger":"helm.controller","msg":"Reconciled release","namespace":"tenant-10","name":"apigroupissue-sample","apiVersion":"agi.symphony.com/v1","kind":"ApiGroupIssue","release":"apigroupissue-sample"}
{"level":"info","ts":1672653346.2418683,"logger":"helm.controller","msg":"Reconciled release","namespace":"tenant-9","name":"apigroupissue-sample","apiVersion":"agi.symphony.com/v1","kind":"ApiGroupIssue","release":"apigroupissue-sample"}
{"level":"info","ts":1672653347.6960278,"logger":"helm.controller","msg":"Reconciled release","namespace":"tenant-5","name":"apigroupissue-sample","apiVersion":"agi.symphony.com/v1","kind":"ApiGroupIssue","release":"apigroupissue-sample"}
{"level":"info","ts":1672653347.7244308,"logger":"helm.controller","msg":"Reconciled release","namespace":"tenant-6","name":"apigroupissue-sample","apiVersion":"agi.symphony.com/v1","kind":"ApiGroupIssue","release":"apigroupissue-sample"}
{"level":"error","ts":1672653349.0529344,"logger":"helm.controller","msg":"Failed to reconcile release","namespace":"tenant-7","name":"apigroupissue-sample","apiVersion":"agi.symphony.com/v1","kind":"ApiGroupIssue","release":"apigroupissue-sample","error":"could not get object: deployments.networking.k8s.io \"tenant-7-api-group-issue-deploy-5\" is forbidden: User \"system:serviceaccount:api-group-issue-helm-op-system:api-group-issue-helm-op-controller-manager\" cannot get resource \"deployments\" in API group \"networking.k8s.io\" in the namespace \"tenant-7\"","stacktrace":"github.com/operator-framework/operator-sdk/internal/helm/controller.HelmOperatorReconciler.Reconcile\n\t/workspace/internal/helm/controller/reconcile.go:363\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.13.0/pkg/internal/controller/controller.go:121\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.13.0/pkg/internal/controller/controller.go:320\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.13.0/pkg/internal/controller/controller.go:273\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.13.0/pkg/internal/controller/controller.go:234"}
{"level":"error","ts":1672653349.0737705,"msg":"Reconciler error","controller":"apigroupissue-controller","object":{"name":"apigroupissue-sample","namespace":"tenant-7"},"namespace":"tenant-7","name":"apigroupissue-sample","reconcileID":"73a22cc6-4c18-4735-9248-c5167bc3055e","error":"could not get object: deployments.networking.k8s.io \"tenant-7-api-group-issue-deploy-5\" is forbidden: User \"system:serviceaccount:api-group-issue-helm-op-system:api-group-issue-helm-op-controller-manager\" cannot get resource \"deployments\" in API group \"networking.k8s.io\" in the namespace \"tenant-7\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.13.0/pkg/internal/controller/controller.go:326\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.13.0/pkg/internal/controller/controller.go:273\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.13.0/pkg/internal/controller/controller.go:234"}
{"level":"info","ts":1672653349.338535,"logger":"helm.controller","msg":"Reconciled release","namespace":"tenant-2","name":"apigroupissue-sample","apiVersion":"agi.symphony.com/v1","kind":"ApiGroupIssue","release":"apigroupissue-sample"}
{"level":"error","ts":1672653350.1464832,"logger":"helm.controller","msg":"Failed to reconcile release","namespace":"tenant-10","name":"apigroupissue-sample","apiVersion":"agi.symphony.com/v1","kind":"ApiGroupIssue","release":"apigroupissue-sample","error":"could not get object: ingresses \"tenant-10-api-group-issue-ing-8\" is forbidden: User \"system:serviceaccount:api-group-issue-helm-op-system:api-group-issue-helm-op-controller-manager\" cannot get resource \"ingresses\" in API group \"\" in the namespace \"tenant-10\"","stacktrace":"github.com/operator-framework/operator-sdk/internal/helm/controller.HelmOperatorReconciler.Reconcile\n\t/workspace/internal/helm/controller/reconcile.go:363\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.13.0/pkg/internal/controller/controller.go:121\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.13.0/pkg/internal/controller/controller.go:320\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.13.0/pkg/internal/controller/controller.go:273\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.13.0/pkg/internal/controller/controller.go:234"}
{"level":"error","ts":1672653350.189844,"msg":"Reconciler error","controller":"apigroupissue-controller","object":{"name":"apigroupissue-sample","namespace":"tenant-10"},"namespace":"tenant-10","name":"apigroupissue-sample","reconcileID":"b6bd09fe-92b5-4b9c-83d5-328b9d1689fd","error":"could not get object: ingresses \"tenant-10-api-group-issue-ing-8\" is forbidden: User \"system:serviceaccount:api-group-issue-helm-op-system:api-group-issue-helm-op-controller-manager\" cannot get resource \"ingresses\" in API group \"\" in the namespace \"tenant-10\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.13.0/pkg/internal/controller/controller.go:326\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.13.0/pkg/internal/controller/controller.go:273\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.13.0/pkg/internal/controller/controller.go:234"}
{"level":"info","ts":1672653350.362542,"logger":"helm.controller","msg":"Reconciled release","namespace":"tenant-8","name":"apigroupissue-sample","apiVersion":"agi.symphony.com/v1","kind":"ApiGroupIssue","release":"apigroupissue-sample"}
{"level":"info","ts":1672653351.2599015,"logger":"helm.controller","msg":"Reconciled release","namespace":"tenant-9","name":"apigroupissue-sample","apiVersion":"agi.symphony.com/v1","kind":"ApiGroupIssue","release":"apigroupissue-sample"}
{"level":"info","ts":1672653351.3180337,"logger":"helm.controller","msg":"Reconciled release","namespace":"tenant-7","name":"apigroupissue-sample","apiVersion":"agi.symphony.com/v1","kind":"ApiGroupIssue","release":"apigroupissue-sample"}
{"level":"error","ts":1672653352.2450194,"logger":"helm.controller","msg":"Failed to reconcile release","namespace":"tenant-7","name":"apigroupissue-sample","apiVersion":"agi.symphony.com/v1","kind":"ApiGroupIssue","release":"apigroupissue-sample","error":"could not get object: services.apps \"tenant-7-api-group-issue-svc-6\" is forbidden: User \"system:serviceaccount:api-group-issue-helm-op-system:api-group-issue-helm-op-controller-manager\" cannot get resource \"services\" in API group \"apps\" in the namespace \"tenant-7\"","stacktrace":"github.com/operator-framework/operator-sdk/internal/helm/controller.HelmOperatorReconciler.Reconcile\n\t/workspace/internal/helm/controller/reconcile.go:363\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.13.0/pkg/internal/controller/controller.go:121\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.13.0/pkg/internal/controller/controller.go:320\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.13.0/pkg/internal/controller/controller.go:273\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.13.0/pkg/internal/controller/controller.go:234"}
{"level":"error","ts":1672653352.264123,"msg":"Reconciler error","controller":"apigroupissue-controller","object":{"name":"apigroupissue-sample","namespace":"tenant-7"},"namespace":"tenant-7","name":"apigroupissue-sample","reconcileID":"f87167d9-5d3f-43ab-90b4-66b6861f15b4","error":"could not get object: services.apps \"tenant-7-api-group-issue-svc-6\" is forbidden: User \"system:serviceaccount:api-group-issue-helm-op-system:api-group-issue-helm-op-controller-manager\" cannot get resource \"services\" in API group \"apps\" in the namespace \"tenant-7\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.13.0/pkg/internal/controller/controller.go:326\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.13.0/pkg/internal/controller/controller.go:273\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.13.0/pkg/internal/controller/controller.go:234"}
{"level":"info","ts":1672653352.6353943,"logger":"helm.controller","msg":"Reconciled release","namespace":"tenant-10","name":"apigroupissue-sample","apiVersion":"agi.symphony.com/v1","kind":"ApiGroupIssue","release":"apigroupissue-sample"}
{"level":"info","ts":1672653353.2589974,"logger":"helm.controller","msg":"Reconciled release","namespace":"tenant-7","name":"apigroupissue-sample","apiVersion":"agi.symphony.com/v1","kind":"ApiGroupIssue","release":"apigroupissue-sample"}
{"level":"info","ts":1672653353.9932454,"logger":"helm.controller","msg":"Reconciled release","namespace":"tenant-10","name":"apigroupissue-sample","apiVersion":"agi.symphony.com/v1","kind":"ApiGroupIssue","release":"apigroupissue-sample"}
{"level":"info","ts":1672653354.502539,"logger":"helm.controller","msg":"Reconciled release","namespace":"tenant-7","name":"apigroupissue-sample","apiVersion":"agi.symphony.com/v1","kind":"ApiGroupIssue","release":"apigroupissue-sample"}
{"level":"info","ts":1672653390.3102448,"logger":"helm.controller","msg":"Reconciled release","namespace":"tenant-3","name":"apigroupissue-sample","apiVersion":"agi.symphony.com/v1","kind":"ApiGroupIssue","release":"apigroupissue-sample"}
{"level":"info","ts":1672653392.0284715,"logger":"helm.controller","msg":"Reconciled release","namespace":"tenant-4","name":"apigroupissue-sample","apiVersion":"agi.symphony.com/v1","kind":"ApiGroupIssue","release":"apigroupissue-sample"}
{"level":"info","ts":1672653392.7302322,"logger":"helm.controller","msg":"Reconciled release","namespace":"tenant-5","name":"apigroupissue-sample","apiVersion":"agi.symphony.com/v1","kind":"ApiGroupIssue","release":"apigroupissue-sample"}
{"level":"info","ts":1672653393.0875666,"logger":"helm.controller","msg":"Reconciled release","namespace":"tenant-2","name":"apigroupissue-sample","apiVersion":"agi.symphony.com/v1","kind":"ApiGroupIssue","release":"apigroupissue-sample"}
{"level":"info","ts":1672653394.228304,"logger":"helm.controller","msg":"Reconciled release","namespace":"tenant-6","name":"apigroupissue-sample","apiVersion":"agi.symphony.com/v1","kind":"ApiGroupIssue","release":"apigroupissue-sample"}
{"level":"info","ts":1672653396.210915,"logger":"helm.controller","msg":"Reconciled release","namespace":"tenant-2","name":"apigroupissue-sample","apiVersion":"agi.symphony.com/v1","kind":"ApiGroupIssue","release":"apigroupissue-sample"}
{"level":"info","ts":1672653396.4399962,"logger":"helm.controller","msg":"Reconciled release","namespace":"tenant-3","name":"apigroupissue-sample","apiVersion":"agi.symphony.com/v1","kind":"ApiGroupIssue","release":"apigroupissue-sample"}
{"level":"error","ts":1672653397.2153714,"logger":"helm.controller","msg":"Failed to reconcile release","namespace":"tenant-4","name":"apigroupissue-sample","apiVersion":"agi.symphony.com/v1","kind":"ApiGroupIssue","release":"apigroupissue-sample","error":"could not get object: ingresses.apps \"tenant-4-api-group-issue-ing-2\" is forbidden: User \"system:serviceaccount:api-group-issue-helm-op-system:api-group-issue-helm-op-controller-manager\" cannot get resource \"ingresses\" in API group \"apps\" in the namespace \"tenant-4\"","stacktrace":"github.com/operator-framework/operator-sdk/internal/helm/controller.HelmOperatorReconciler.Reconcile\n\t/workspace/internal/helm/controller/reconcile.go:363\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.13.0/pkg/internal/controller/controller.go:121\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.13.0/pkg/internal/controller/controller.go:320\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.13.0/pkg/internal/controller/controller.go:273\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.13.0/pkg/internal/controller/controller.go:234"}
{"level":"error","ts":1672653397.2456267,"msg":"Reconciler error","controller":"apigroupissue-controller","object":{"name":"apigroupissue-sample","namespace":"tenant-4"},"namespace":"tenant-4","name":"apigroupissue-sample","reconcileID":"35fe3b58-2b89-4b46-8a76-64ca3b85c620","error":"could not get object: ingresses.apps \"tenant-4-api-group-issue-ing-2\" is forbidden: User \"system:serviceaccount:api-group-issue-helm-op-system:api-group-issue-helm-op-controller-manager\" cannot get resource \"ingresses\" in API group \"apps\" in the namespace \"tenant-4\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.13.0/pkg/internal/controller/controller.go:326\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.13.0/pkg/internal/controller/controller.go:273\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.13.0/pkg/internal/controller/controller.go:234"}
{"level":"info","ts":1672653397.4344475,"logger":"helm.controller","msg":"Reconciled release","namespace":"tenant-8","name":"apigroupissue-sample","apiVersion":"agi.symphony.com/v1","kind":"ApiGroupIssue","release":"apigroupissue-sample"}
{"level":"error","ts":1672653398.0753016,"logger":"helm.controller","msg":"Failed to reconcile release","namespace":"tenant-3","name":"apigroupissue-sample","apiVersion":"agi.symphony.com/v1","kind":"ApiGroupIssue","release":"apigroupissue-sample","error":"could not get object: services.apps \"tenant-3-api-group-issue-svc-5\" is forbidden: User \"system:serviceaccount:api-group-issue-helm-op-system:api-group-issue-helm-op-controller-manager\" cannot get resource \"services\" in API group \"apps\" in the namespace \"tenant-3\"","stacktrace":"github.com/operator-framework/operator-sdk/internal/helm/controller.HelmOperatorReconciler.Reconcile\n\t/workspace/internal/helm/controller/reconcile.go:363\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.13.0/pkg/internal/controller/controller.go:121\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.13.0/pkg/internal/controller/controller.go:320\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.13.0/pkg/internal/controller/controller.go:273\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.13.0/pkg/internal/controller/controller.go:234"}
{"level":"error","ts":1672653398.092799,"msg":"Reconciler error","controller":"apigroupissue-controller","object":{"name":"apigroupissue-sample","namespace":"tenant-3"},"namespace":"tenant-3","name":"apigroupissue-sample","reconcileID":"9c917f28-7904-4a8e-8a9e-c0498cbce83c","error":"could not get object: services.apps \"tenant-3-api-group-issue-svc-5\" is forbidden: User \"system:serviceaccount:api-group-issue-helm-op-system:api-group-issue-helm-op-controller-manager\" cannot get resource \"services\" in API group \"apps\" in the namespace \"tenant-3\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.13.0/pkg/internal/controller/controller.go:326\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.13.0/pkg/internal/controller/controller.go:273\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.13.0/pkg/internal/controller/controller.go:234"}
{"level":"info","ts":1672653398.4369118,"logger":"helm.controller","msg":"Reconciled release","namespace":"tenant-2","name":"apigroupissue-sample","apiVersion":"agi.symphony.com/v1","kind":"ApiGroupIssue","release":"apigroupissue-sample"}
{"level":"info","ts":1672653398.9274225,"logger":"helm.controller","msg":"Reconciled release","namespace":"tenant-9","name":"apigroupissue-sample","apiVersion":"agi.symphony.com/v1","kind":"ApiGroupIssue","release":"apigroupissue-sample"}
{"level":"info","ts":1672653399.6042168,"logger":"helm.controller","msg":"Reconciled release","namespace":"tenant-4","name":"apigroupissue-sample","apiVersion":"agi.symphony.com/v1","kind":"ApiGroupIssue","release":"apigroupissue-sample"}
{"level":"info","ts":1672653399.9605913,"logger":"helm.controller","msg":"Reconciled release","namespace":"tenant-3","name":"apigroupissue-sample","apiVersion":"agi.symphony.com/v1","kind":"ApiGroupIssue","release":"apigroupissue-sample"}
{"level":"info","ts":1672653400.721334,"logger":"helm.controller","msg":"Reconciled release","namespace":"tenant-2","name":"apigroupissue-sample","apiVersion":"agi.symphony.com/v1","kind":"ApiGroupIssue","release":"apigroupissue-sample"}
{"level":"info","ts":1672653401.0330744,"logger":"helm.controller","msg":"Reconciled release","namespace":"tenant-4","name":"apigroupissue-sample","apiVersion":"agi.symphony.com/v1","kind":"ApiGroupIssue","release":"apigroupissue-sample"}
{"level":"info","ts":1672653401.680854,"logger":"helm.controller","msg":"Reconciled release","namespace":"tenant-3","name":"apigroupissue-sample","apiVersion":"agi.symphony.com/v1","kind":"ApiGroupIssue","release":"apigroupissue-sample"}
{"level":"info","ts":1672653402.1116257,"logger":"helm.controller","msg":"Reconciled release","namespace":"tenant-2","name":"apigroupissue-sample","apiVersion":"agi.symphony.com/v1","kind":"ApiGroupIssue","release":"apigroupissue-sample"}
{"level":"info","ts":1672653402.369346,"logger":"helm.controller","msg":"Reconciled release","namespace":"tenant-4","name":"apigroupissue-sample","apiVersion":"agi.symphony.com/v1","kind":"ApiGroupIssue","release":"apigroupissue-sample"}
{"level":"info","ts":1672653403.4723146,"logger":"helm.controller","msg":"Reconciled release","namespace":"tenant-4","name":"apigroupissue-sample","apiVersion":"agi.symphony.com/v1","kind":"ApiGroupIssue","release":"apigroupissue-sample"}
{"level":"info","ts":1672653403.5516784,"logger":"helm.controller","msg":"Reconciled release","namespace":"tenant-2","name":"apigroupissue-sample","apiVersion":"agi.symphony.com/v1","kind":"ApiGroupIssue","release":"apigroupissue-sample"}
{"level":"info","ts":1672653404.470545,"logger":"helm.controller","msg":"Reconciled release","namespace":"tenant-4","name":"apigroupissue-sample","apiVersion":"agi.symphony.com/v1","kind":"ApiGroupIssue","release":"apigroupissue-sample"}