YUNIKORN-2151 Report resource used by placeholder pods in the app su…

pkg/scheduler/objects/application.go

@@ -87,8 +87,9 @@
 	user              security.UserGroup        // owner of the application
 	allocatedResource *resources.Resource       // total allocated resources
 
-	usedResource      *resources.TrackedResource // keep track of resource usage of the application
-	preemptedResource *resources.TrackedResource // keep track of preempted resource usage of the application
+	usedResource        *resources.TrackedResource // keep track of resource usage of the application
+	preemptedResource   *resources.TrackedResource // keep track of preempted resource usage of the application
+	placeholderResource *resources.TrackedResource // keep track of placeholder resource usage of the application
 
 	maxAllocatedResource *resources.Resource         // max allocated resources
 	allocatedPlaceholder *resources.Resource         // total allocated placeholder resources
@@ -118,16 +119,17 @@
 }
 
 type ApplicationSummary struct {
-	ApplicationID     string
-	SubmissionTime    time.Time
-	StartTime         time.Time
-	FinishTime        time.Time
-	User              string
-	Queue             string
-	State             string
-	RmID              string
-	ResourceUsage     *resources.TrackedResource
-	PreemptedResource *resources.TrackedResource
+	ApplicationID       string
+	SubmissionTime      time.Time
+	StartTime           time.Time
+	FinishTime          time.Time
+	User                string
+	Queue               string
+	State               string
+	RmID                string
+	ResourceUsage       *resources.TrackedResource
+	PreemptedResource   *resources.TrackedResource
+	PlaceHolderResource *resources.TrackedResource
 }
 
 func (as *ApplicationSummary) DoLogging() {
@@ -142,26 +144,29 @@
 		zap.String("rmID", as.RmID),
 		zap.Any("resourceUsage", as.ResourceUsage.TrackedResourceMap),
 		zap.Any("preemptedResource", as.PreemptedResource.TrackedResourceMap),
+		zap.Any("placeHolderResource", as.PlaceHolderResource.TrackedResourceMap),
 	)
 }
 
 func (sa *Application) GetApplicationSummary(rmID string) *ApplicationSummary {
 	sa.RLock()
 	defer sa.RUnlock()
 	state := sa.stateMachine.Current()
-	ru := sa.usedResource.Clone()
-	pu := sa.preemptedResource.Clone()
+	resourceUsage := sa.usedResource.Clone()
+	preemptedUsage := sa.preemptedResource.Clone()
+	placeHolderUsage := sa.placeholderResource.Clone()
 	appSummary := &ApplicationSummary{
-		ApplicationID:     sa.ApplicationID,
-		SubmissionTime:    sa.SubmissionTime,
-		StartTime:         sa.startTime,
-		FinishTime:        sa.finishedTime,
-		User:              sa.user.User,
-		Queue:             sa.queuePath,
-		State:             state,
-		RmID:              rmID,
-		ResourceUsage:     ru,
-		PreemptedResource: pu,
+		ApplicationID:       sa.ApplicationID,
+		SubmissionTime:      sa.SubmissionTime,
+		StartTime:           sa.startTime,
+		FinishTime:          sa.finishedTime,
+		User:                sa.user.User,
+		Queue:               sa.queuePath,
+		State:               state,
+		RmID:                rmID,
+		ResourceUsage:       resourceUsage,
+		PreemptedResource:   preemptedUsage,
+		PlaceHolderResource: placeHolderUsage,
 	}
 	return appSummary
 }
@@ -177,6 +182,7 @@
 		allocatedResource:     resources.NewResource(),
 		usedResource:          resources.NewTrackedResource(),
 		preemptedResource:     resources.NewTrackedResource(),
+		placeholderResource:   resources.NewTrackedResource(),
 		maxAllocatedResource:  resources.NewResource(),
 		allocatedPlaceholder:  resources.NewResource(),
 		requests:              make(map[string]*AllocationAsk),
@@ -1684,6 +1690,8 @@
 func (sa *Application) trackCompletedResource(info *Allocation) {
 	if info.IsPreempted() {
 		sa.updatePreemptedResource(info)
+	} else if info.IsPlaceholder() {
+		sa.updatePlaceholderResource(info)
 	} else {
 		sa.updateUsedResource(info)
 	}
@@ -1696,6 +1704,13 @@
 		info.GetAllocatedResource(), info.GetBindTime())
 }
 
+// When the placeholder allocated with this allocation is to be removed,
+// have the placeholderResource to aggregate the resource used by this allocation
+func (sa *Application) updatePlaceholderResource(info *Allocation) {
+	sa.placeholderResource.AggregateTrackedResource(info.GetInstanceType(),
+		info.GetAllocatedResource(), info.GetBindTime())
+}
+
 // When the resource allocated with this allocation is to be preempted,
 // have the preemptedResource to aggregate the resource used by this allocation
 func (sa *Application) updatePreemptedResource(info *Allocation) {
@@ -1791,6 +1806,9 @@
 				eventWarning = "Application state not changed while removing a placeholder allocation"
 			}
 		}
+		// Aggregate the resources used by this alloc to the application's resource tracker
+		sa.trackCompletedResource(alloc)
+
 		sa.decUserResourceUsage(alloc.GetAllocatedResource(), removeApp)
 	} else {
 		sa.allocatedResource = resources.Sub(sa.allocatedResource, alloc.GetAllocatedResource())
@@ -2033,6 +2051,7 @@
 
 func (sa *Application) cleanupTrackedResource() {
 	sa.usedResource = nil
+	sa.placeholderResource = nil
 	sa.preemptedResource = nil
 }
 
@@ -2049,6 +2068,8 @@
 	appSummary := sa.GetApplicationSummary(rmID)
 	appSummary.DoLogging()
 	appSummary.ResourceUsage = nil
+	appSummary.PreemptedResource = nil
+	appSummary.PlaceHolderResource = nil
 }
 
 func (sa *Application) HasPlaceholderAllocation() bool {

pkg/scheduler/objects/application_test.go

@@ -1067,6 +1067,13 @@ func assertResourceUsage(t *testing.T, appSummary *ApplicationSummary, memorySec
 	assert.Equal(t, vcoresSecconds, detailedResource["vcores"])
 }
 
+func assertPlaceHolderResource(t *testing.T, appSummary *ApplicationSummary, memorySeconds int64,
+	vcoresSecconds int64) {
+	detailedResource := appSummary.PlaceHolderResource.TrackedResourceMap[instType1]
+	assert.Equal(t, memorySeconds, detailedResource["memory"])
+	assert.Equal(t, vcoresSecconds, detailedResource["vcores"])
+}
+
 func TestResourceUsageAggregation(t *testing.T) {
 	setupUGM()
 
@@ -1332,6 +1339,9 @@ func TestReplaceAllocationTracking(t *testing.T) {
 	ph1 := newPlaceholderAlloc(appID1, "uuid-1", nodeID1, res)
 	ph2 := newPlaceholderAlloc(appID1, "uuid-2", nodeID1, res)
 	ph3 := newPlaceholderAlloc(appID1, "uuid-3", nodeID1, res)
+	ph1.SetInstanceType(instType1)
+	ph2.SetInstanceType(instType1)
+	ph3.SetInstanceType(instType1)
 	app.AddAllocation(ph1)
 	assert.NilError(t, err, "could not add ask")
 	app.addPlaceholderDataWithLocking(ph1.GetAsk())
@@ -1343,6 +1353,9 @@ func TestReplaceAllocationTracking(t *testing.T) {
 	assert.NilError(t, err, "could not add ask")
 	app.addPlaceholderDataWithLocking(ph3.GetAsk())
 
+	// set time to 1 second, in order to test placeholder resource usage
+	time.Sleep(1 * time.Second)
+
 	// replace placeholders
 	realAlloc1 := newAllocation(appID1, "uuid-100", nodeID1, res)
 	realAlloc1.SetResult(Replaced)
@@ -1365,6 +1378,10 @@ func TestReplaceAllocationTracking(t *testing.T) {
 	app.RemoveAllocation("uuid-3", si.TerminationType_PLACEHOLDER_REPLACED)
 	assert.Equal(t, "uuid-3", alloc.uuid)
 	assert.Equal(t, false, app.HasPlaceholderAllocation())
+
+	// check placeholder resource usage
+	appSummary := app.GetApplicationSummary("default")
+	assertPlaceHolderResource(t, appSummary, 300, 30)
 }
 
 func TestTimeoutPlaceholderSoftStyle(t *testing.T) {