formalize RecentApp, report in throttler status

Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com>

go/vt/vtctl/grpcvtctldserver/server.go

@@ -767,6 +767,7 @@ func (s *VtctldServer) GetThrottlerStatus(ctx context.Context, req *vtctldatapb.
 		MetricsHealth:           make(map[string]*vtctldatapb.GetThrottlerStatusResponse_MetricHealth),
 		ThrottledApps:           r.ThrottledApps,
 		AppCheckedMetrics:       r.AppCheckedMetrics,
+		RecentApps:              make(map[string]*vtctldatapb.GetThrottlerStatusResponse_RecentApp),
 	}
 	for k, m := range r.AggregatedMetrics {
 		resp.AggregatedMetrics[k] = &vtctldatapb.GetThrottlerStatusResponse_MetricResult{
@@ -780,6 +781,12 @@ func (s *VtctldServer) GetThrottlerStatus(ctx context.Context, req *vtctldatapb.
 			SecondsSinceLastHealthy: m.SecondsSinceLastHealthy,
 		}
 	}
+	for k, a := range r.RecentApps {
+		resp.RecentApps[k] = &vtctldatapb.GetThrottlerStatusResponse_RecentApp{
+			CheckedAt:  a.CheckedAt,
+			StatusCode: a.StatusCode,
+		}
+	}
 	return resp, nil
 }
 

go/vt/vttablet/tabletmanager/rpc_throttler.go

@@ -107,6 +107,7 @@ func (tm *TabletManager) GetThrottlerStatus(ctx context.Context, req *tabletmana
 		MetricsHealth:           make(map[string]*tabletmanagerdatapb.GetThrottlerStatusResponse_MetricHealth),
 		ThrottledApps:           make(map[string]*topodatapb.ThrottledAppRule),
 		AppCheckedMetrics:       status.AppCheckedMetrics,
+		RecentApps:              make(map[string]*tabletmanagerdatapb.GetThrottlerStatusResponse_RecentApp),
 	}
 	for k, m := range status.AggregatedMetrics {
 		val, err := m.Get()
@@ -131,5 +132,11 @@ func (tm *TabletManager) GetThrottlerStatus(ctx context.Context, req *tabletmana
 			Exempt:    app.Exempt,
 		}
 	}
+	for _, recentApp := range status.RecentApps {
+		resp.RecentApps[recentApp.AppName] = &tabletmanagerdatapb.GetThrottlerStatusResponse_RecentApp{
+			CheckedAt:  protoutil.TimeToProto(recentApp.CheckedAt),
+			StatusCode: int32(recentApp.StatusCode),
+		}
+	}
 	return resp, nil
 }

go/vt/vttablet/tabletserver/throttle/base/recent_app.go

@@ -46,15 +46,16 @@ import (
 
 // RecentApp indicates when an app was last checked
 type RecentApp struct {
-	CheckedAtEpoch      int64
-	MinutesSinceChecked int64
+	AppName    string
+	CheckedAt  time.Time
+	StatusCode int
 }
 
 // NewRecentApp creates a RecentApp
-func NewRecentApp(checkedAt time.Time) *RecentApp {
+func NewRecentApp(appName string, statusCode int) *RecentApp {
 	result := &RecentApp{
-		CheckedAtEpoch:      checkedAt.Unix(),
-		MinutesSinceChecked: int64(time.Since(checkedAt).Minutes()),
+		AppName:   appName,
+		CheckedAt: time.Now(),
 	}
 	return result
 }

go/vt/vttablet/tabletserver/throttle/check.go

@@ -190,7 +190,6 @@ func (check *ThrottlerCheck) Check(ctx context.Context, appName string, scope ba
 		}
 
 		metricCheckResult := check.checkAppMetricResult(ctx, appName, metricResultFunc, flags)
-		check.throttler.markRecentApp(appName)
 		if !throttlerapp.VitessName.Equals(appName) {
 			go func(statusCode int) {
 				if metricScope == base.UndefinedScope {
@@ -241,6 +240,7 @@ func (check *ThrottlerCheck) Check(ctx context.Context, appName string, scope ba
 			statsThrottlerCheckAnyError.Add(1)
 		}
 	}(checkResult.StatusCode)
+	go check.throttler.markRecentApp(appName, checkResult.StatusCode)
 	return checkResult
 }
 

go/vt/vttablet/tabletserver/throttle/throttler.go

@@ -93,7 +93,7 @@ const (
 	recentCheckRateLimiterInterval = 1 * time.Second // Ticker assisting in determining when the throttler was last checked
 
 	aggregatedMetricsExpiration = 5 * time.Second
-	recentAppsExpiration        = time.Hour * 24
+	recentAppsExpiration        = time.Hour
 
 	nonDeprioritizedAppMapExpiration = time.Second
 
@@ -237,6 +237,7 @@ type ThrottlerStatus struct {
 	MetricsHealth     base.MetricHealthMap
 	ThrottledApps     []base.AppThrottle
 	AppCheckedMetrics map[string]string
+	RecentApps        map[string](*base.RecentApp)
 }
 
 // NewThrottler creates a Throttler
@@ -263,7 +264,7 @@ func NewThrottler(env tabletenv.Env, srvTopoServer srvtopo.Server, ts *topo.Serv
 	throttler.throttledApps = cache.New(cache.NoExpiration, 0)
 	throttler.mysqlMetricThresholds = cache.New(cache.NoExpiration, 0)
 	throttler.aggregatedMetrics = cache.New(aggregatedMetricsExpiration, 0)
-	throttler.recentApps = cache.New(recentAppsExpiration, 0)
+	throttler.recentApps = cache.New(recentAppsExpiration, recentAppsExpiration)
 	throttler.metricsHealth = cache.New(cache.NoExpiration, 0)
 	throttler.appCheckedMetrics = cache.New(cache.NoExpiration, 0)
 	throttler.nonLowPriorityAppRequestsThrottled = cache.New(nonDeprioritizedAppMapExpiration, 0)
@@ -1482,16 +1483,17 @@ func (throttler *Throttler) ThrottledAppsMap() (result map[string](*base.AppThro
 }
 
 // markRecentApp takes note that an app has just asked about throttling, making it "recent"
-func (throttler *Throttler) markRecentApp(appName string) {
-	throttler.recentApps.Set(appName, time.Now(), cache.DefaultExpiration)
+func (throttler *Throttler) markRecentApp(appName string, statusCode int) {
+	recentApp := base.NewRecentApp(appName, statusCode)
+	throttler.recentApps.Set(appName, recentApp, cache.DefaultExpiration)
 }
 
-// RecentAppsMap returns a (copy) map of apps which checked for throttling recently
-func (throttler *Throttler) RecentAppsMap() (result map[string](*base.RecentApp)) {
+// recentAppsSnapshot returns a (copy) map of apps which checked for throttling recently
+func (throttler *Throttler) recentAppsSnapshot() (result map[string](*base.RecentApp)) {
 	result = make(map[string](*base.RecentApp))
 
 	for recentAppKey, item := range throttler.recentApps.Items() {
-		recentApp := base.NewRecentApp(item.Object.(time.Time))
+		recentApp := item.Object.(*base.RecentApp)
 		result[recentAppKey] = recentApp
 	}
 	return result
@@ -1657,5 +1659,6 @@ func (throttler *Throttler) Status() *ThrottlerStatus {
 		MetricsHealth:     throttler.metricsHealthSnapshot(),
 		ThrottledApps:     throttler.ThrottledApps(),
 		AppCheckedMetrics: throttler.appCheckedMetricsSnapshot(),
+		RecentApps:        throttler.recentAppsSnapshot(),
 	}
 }