balloons/agent: fix/rewrite/simplify extended resource handling

cmd/plugins/balloons/policy/balloons-policy.go

@@ -579,13 +579,15 @@ func (p *balloons) GetTopologyZones() []*policy.TopologyZone {
 }
 
 // GetExtendedResources returns the node-level extended resources
-// the balloons policy publishes for the local Node.
-func (p *balloons) GetExtendedResources() map[string]resource.Quantity {
-	out := map[string]resource.Quantity{}
-	if p.cpuClasses == nil || !p.cpuClasses.PctActive() {
-		return out
-	}
-	if p.bpoptions == nil {
+// the balloons policy manages on the local Node.
+func (p *balloons) GetExtendedResources() map[string]*resource.Quantity {
+	// Own the entire domain unconditionally: any key under it
+	// that we do not explicitly publish below must be removed.
+	out := map[string]*resource.Quantity{
+		"cpuclass.balloons.nri.io/*": nil,
+	}
+	// Experimental cpuClass.PublishExtendedResource publishes only PCT capacity.
+	if p.cpuClasses == nil || !p.cpuClasses.PctActive() || p.bpoptions == nil {
 		return out
 	}
 	for _, cc := range p.bpoptions.CPUClasses {
@@ -607,7 +609,7 @@ func (p *balloons) GetExtendedResources() map[string]resource.Quantity {
 		if free < 0 {
 			free = 0
 		}
-		out["cpuclass.balloons.nri.io/"+cc.Name] = *resource.NewQuantity(int64(free), resource.DecimalSI)
+		out["cpuclass.balloons.nri.io/"+cc.Name] = resource.NewQuantity(int64(free), resource.DecimalSI)
 	}
 	return out
 }

cmd/plugins/template/policy/template-policy.go

@@ -122,7 +122,7 @@ func (p *policy) GetTopologyZones() []*policyapi.TopologyZone {
 
 // GetExtendedResources returns the node-level extended resources
 // to publish for this policy. The template policy publishes none.
-func (p *policy) GetExtendedResources() map[string]resource.Quantity {
+func (p *policy) GetExtendedResources() map[string]*resource.Quantity {
 	return nil
 }
 

cmd/plugins/topology-aware/policy/topology-aware-policy.go

@@ -397,7 +397,7 @@ func (p *policy) GetTopologyZones() []*policyapi.TopologyZone {
 
 // GetExtendedResources returns the node-level extended resources
 // to publish for this policy. The topology-aware policy publishes none.
-func (p *policy) GetExtendedResources() map[string]resource.Quantity {
+func (p *policy) GetExtendedResources() map[string]*resource.Quantity {
 	return nil
 }
 

docs/resource-policy/policy/balloons.md

@@ -910,9 +910,11 @@ CPU class definitions. Each class is an object with:
   available CPU capacity as
   `cpuclass.balloons.nri.io/<CPU-CLASS-NAME>` extended
   resource. Enables Kubernetes to schedule pods on nodes with enough
-  CPUs of wanted priority. Note: container's `cpu` and
-  extended resource requests **must be equal** to avoid
-  over and under subscription.
+  CPUs of wanted priority. Notes: container's `cpu` and extended
+  resource requests **must be equal** to avoid over and under
+  subscription. Extended resources are not cleaned up from node status
+  when balloons is stopped or uninstalled. Balloons reconciliation
+  removes extended resources when configured not to publish them.
 
 **`turboDomain`** (string, policy-level configuration):
 

docs/resource-policy/policy/howto/balloons-pct-example-auto.md

@@ -796,10 +796,24 @@ kubectl delete -f pod-hp-1.yaml -f pod-hp-2.yaml -f pod-hp-3.yaml \
 kubectl delete -f balloons-pct-managed.yaml --ignore-not-found
 ```
 
-Deleting the `BalloonsPolicy` CR is the policy's defined "reset"
-trigger: the plugin reacts to losing its effective configuration
-by removing every `cpuclass.balloons.nri.io/*` extended resource
-it had published. Verify before uninstalling the chart:
+Deleting the `BalloonsPolicy` CR, or uninstalling the chart, does
+not remove published `cpuclass.balloons.nri.io/*` extended
+resources by itself: they live in the node's status capacity, not
+in CR-managed objects, and the plugin does not clear them on
+shutdown. A running plugin reconciles them instead: whenever the
+balloons policy is (re)configured so that it no longer publishes a
+resource, the agent removes the stale resource; likewise, starting
+the plugin with a non-publishing configuration reconciles away any
+resource left over from a previous run, regardless of how it got
+there.
+
+It is therefore good practice to apply a "reset" `BalloonsPolicy`
+(one that publishes nothing) and let the plugin reconcile before
+uninstalling: this also clears any published
+`cpuclass.balloons.nri.io/*` extended resources while the plugin is
+still running.
+
+To verify no `cpuclass.balloons.nri.io/*` resources remain:
 
 ```bash
 kubectl get node -o jsonpath='{.items[0].status.capacity}' \

docs/resource-policy/policy/howto/balloons-pct-example-manual.md

@@ -804,10 +804,24 @@ kubectl delete -f pod-hp-1.yaml -f pod-hp-2.yaml -f pod-hp-3.yaml \
 kubectl delete -f balloons-pct-assoconly.yaml --ignore-not-found
 ```
 
-Deleting the `BalloonsPolicy` CR is the policy's defined "reset"
-trigger: the plugin reacts to losing its effective configuration
-by removing every `cpuclass.balloons.nri.io/*` extended resource
-it had published. Verify before uninstalling the chart:
+Deleting the `BalloonsPolicy` CR, or uninstalling the chart, does
+not remove published `cpuclass.balloons.nri.io/*` extended
+resources by itself: they live in the node's status capacity, not
+in CR-managed objects, and the plugin does not clear them on
+shutdown. A running plugin reconciles them instead: whenever the
+balloons policy is (re)configured so that it no longer publishes a
+resource, the agent removes the stale resource; likewise, starting
+the plugin with a non-publishing configuration reconciles away any
+resource left over from a previous run, regardless of how it got
+there.
+
+It is therefore good practice to apply a "reset" `BalloonsPolicy`
+(one that publishes nothing) and let the plugin reconcile before
+uninstalling: this also clears any published
+`cpuclass.balloons.nri.io/*` extended resources while the plugin is
+still running.
+
+To verify no `cpuclass.balloons.nri.io/*` resources remain:
 
 ```bash
 kubectl get node -o jsonpath='{.items[0].status.capacity}' \

pkg/agent/agent.go

@@ -23,6 +23,7 @@ import (
 	"sync"
 
 	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
@@ -143,6 +144,10 @@ type Agent struct {
 	groupCfg      metav1.Object   // group-specific/default config resource
 	currentCfg    metav1.Object
 
+	extResWant map[string]*resource.Quantity // extended resources desired state
+	extResLock sync.Mutex                    // lock accessing the desired state
+	extResWake chan struct{}                 // reconciliation trigger
+
 	stopLock sync.Mutex
 	stopC    chan struct{}
 	doneC    chan struct{}
@@ -161,6 +166,7 @@ func New(cfgIf ConfigInterface, options ...Option) (*Agent, error) {
 		namespace:  defaultNamespace,
 		groupLabel: defaultGroupLabel,
 		cfgIf:      cfgIf,
+		extResWake: make(chan struct{}, 1),
 		stopC:      make(chan struct{}),
 	}
 
@@ -201,6 +207,8 @@ func (a *Agent) Start(notifyFn NotifyFn) error {
 		return w.ResultChan()
 	}
 
+	go a.reconcileExtendedResourcesLoop()
+
 	for {
 		select {
 		case <-a.stopC:
@@ -258,10 +266,6 @@ func (a *Agent) Stop() {
 	defer a.stopLock.Unlock()
 
 	if a.stopC != nil {
-		// Remove any extended resources we own on this node so
-		// a graceful shutdown does not leave orphan capacity
-		// entries behind.
-		a.ClearNodeExtendedResources()
 		close(a.stopC)
 		<-a.doneC
 		a.stopC = nil
@@ -601,10 +605,6 @@ func (a *Agent) updateGroupConfig(obj runtime.Object) {
 func (a *Agent) updateConfig(cfg metav1.Object) {
 	if cfg == nil {
 		log.Warnf("node (%s) has no effective configuration", a.nodeName)
-		// With no effective configuration there is nothing left to
-		// publish, so drop any extended resources we currently own
-		// on the node.
-		a.ClearNodeExtendedResources()
 		return
 	}