Add rate limit toggles

cache/credentials_cache.go

@@ -14,6 +14,8 @@
 package cache
 
 import (
+	"time"
+
 	"github.com/cloudbase/garm/params"
 )
 
@@ -52,6 +54,11 @@ func (g *credentialCache) UpdateCredentialsUsingEndpoint(ep params.ForgeEndpoint
 
 func (g *credentialCache) SetCredentials(credentials params.ForgeCredentials) {
 	g.Update(func(cache map[uint]params.ForgeCredentials) {
+		// Credentials sourced from the database carry no rate limit info, so
+		// an update clears the recorded values. That is fine as an update may
+		// be a token swap, and rate limits are per token. The clients record
+		// fresh values on every forge response and the cache worker's rate
+		// limit loop repolls within 30 seconds either way.
 		cache[credentials.ID] = credentials
 		UpdateCredentialsInAffectedEntities(credentials)
 	})
@@ -110,3 +117,52 @@ func GetAllGiteaCredentialsAsMap() map[uint]params.ForgeCredentials {
 func UpdateCredentialsUsingEndpoint(ep params.ForgeEndpoint) {
 	giteaCredentialsCache.UpdateCredentialsUsingEndpoint(ep)
 }
+
+// GetForgeCredentials returns the cached credentials with the given ID for
+// the specified forge type. Unlike the copies workers hold on their entities,
+// the cached credentials carry the most recently observed rate limit values.
+func GetForgeCredentials(forgeType params.EndpointType, id uint) (params.ForgeCredentials, bool) {
+	switch forgeType {
+	case params.GithubEndpointType:
+		return GetGithubCredentials(id)
+	case params.GiteaEndpointType:
+		return GetGiteaCredentials(id)
+	}
+	return params.ForgeCredentials{}, false
+}
+
+// entityCredentials resolves the freshest credentials for an entity: the
+// entity cache tracks credential swaps, and the credentials cache carries
+// the most recently observed rate limit values. Workers hold set-once
+// copies of both, so rate limit checks must go through here.
+func entityCredentials(entityID string) (params.ForgeCredentials, bool) {
+	entity, ok := GetEntity(entityID)
+	if !ok {
+		return params.ForgeCredentials{}, false
+	}
+	return GetForgeCredentials(entity.Credentials.ForgeType, entity.Credentials.ID)
+}
+
+// EntityRateLimitReached reports whether the credentials currently assigned
+// to the given entity should be considered rate limited for normal
+// (non-critical) operations, and when the quota resets. Entities or
+// credentials missing from the cache are never limited.
+func EntityRateLimitReached(entityID string) (bool, time.Time) {
+	creds, ok := entityCredentials(entityID)
+	if !ok {
+		return false, time.Time{}
+	}
+	return creds.RateLimitReached()
+}
+
+// EntityRateLimitExhausted reports whether the quota of the credentials
+// currently assigned to the given entity is fully spent, meaning even
+// critical operations cannot succeed, and when it resets. Entities or
+// credentials missing from the cache are never limited.
+func EntityRateLimitExhausted(entityID string) (bool, time.Time) {
+	creds, ok := entityCredentials(entityID)
+	if !ok {
+		return false, time.Time{}
+	}
+	return creds.CriticalRateLimitReached()
+}

cmd/garm-cli/cmd/github_credentials.go

@@ -38,6 +38,8 @@ var (
 	credentialsPrivateKeyPath    string
 	credentialsType              string
 	credentialsEndpoint          string
+	credentialsReserveEnabled    bool
+	credentialsReservePercent    int
 )
 
 // credentialsCmd represents the credentials command
@@ -117,7 +119,7 @@ var githubCredentialsUpdateCmd = &cobra.Command{
 	Short:        "Update a github credential",
 	Long:         "Update a github credential",
 	SilenceUsage: true,
-	RunE: func(_ *cobra.Command, args []string) error {
+	RunE: func(cmd *cobra.Command, args []string) error {
 		if needsInit {
 			return errNeedsInitError
 		}
@@ -135,7 +137,7 @@ var githubCredentialsUpdateCmd = &cobra.Command{
 			return fmt.Errorf("invalid credential ID: %s", args[0])
 		}
 
-		updateParams, err := parseCredentialsUpdateParams()
+		updateParams, err := parseCredentialsUpdateParams(cmd)
 		if err != nil {
 			return err
 		}
@@ -222,6 +224,8 @@ func init() {
 	githubCredentialsUpdateCmd.Flags().Int64Var(&credentialsAppInstallationID, "app-installation-id", 0, "If the credential is an app, the installation ID")
 	githubCredentialsUpdateCmd.Flags().Int64Var(&credentialsAppID, "app-id", 0, "If the credential is an app, the app ID")
 	githubCredentialsUpdateCmd.Flags().StringVar(&credentialsPrivateKeyPath, "private-key-path", "", "If the credential is an app, the path to the private key file")
+	githubCredentialsUpdateCmd.Flags().BoolVar(&credentialsReserveEnabled, "reserve-usage-enabled", false, "Reserve a percentage of the rate limit for critical operations (such as runner deletion)")
+	githubCredentialsUpdateCmd.Flags().IntVar(&credentialsReservePercent, "reserve-usage-percentage", 0, "Percentage of the rate limit to reserve for critical operations (0-100). A value between 5 and 20 should be safe on most setups")
 
 	githubCredentialsListCmd.Flags().BoolVarP(&long, "long", "l", false, "Include additional info.")
 
@@ -238,6 +242,8 @@ func init() {
 	githubCredentialsAddCmd.Flags().StringVar(&credentialsPrivateKeyPath, "private-key-path", "", "If the credential is an app, the path to the private key file")
 	githubCredentialsAddCmd.Flags().StringVar(&credentialsType, "auth-type", "", "The type of the credential")
 	githubCredentialsAddCmd.Flags().StringVar(&credentialsEndpoint, "endpoint", "", "The endpoint to associate the credential with")
+	githubCredentialsAddCmd.Flags().BoolVar(&credentialsReserveEnabled, "reserve-usage-enabled", false, "Reserve a percentage of the rate limit for critical operations (such as runner deletion)")
+	githubCredentialsAddCmd.Flags().IntVar(&credentialsReservePercent, "reserve-usage-percentage", 0, "Percentage of the rate limit to reserve for critical operations (0-100). A value between 5 and 20 should be safe on most setups")
 
 	githubCredentialsAddCmd.MarkFlagsMutuallyExclusive("pat-oauth-token", "app-installation-id")
 	githubCredentialsAddCmd.MarkFlagsMutuallyExclusive("pat-oauth-token", "app-id")
@@ -285,6 +291,8 @@ func parseCredentialsAddParams() (ret params.CreateGithubCredentialsParams, err
 	ret.Description = credentialsDescription
 	ret.AuthType = params.ForgeAuthType(credentialsType)
 	ret.Endpoint = credentialsEndpoint
+	ret.ReserveUsageEnabled = credentialsReserveEnabled
+	ret.ReserveUsagePercentage = credentialsReservePercent
 	switch ret.AuthType {
 	case params.ForgeAuthTypePAT:
 		ret.PAT.OAuth2Token = credentialsOAuthToken
@@ -303,7 +311,7 @@ func parseCredentialsAddParams() (ret params.CreateGithubCredentialsParams, err
 	return ret, nil
 }
 
-func parseCredentialsUpdateParams() (params.UpdateGithubCredentialsParams, error) {
+func parseCredentialsUpdateParams(cmd *cobra.Command) (params.UpdateGithubCredentialsParams, error) {
 	var updateParams params.UpdateGithubCredentialsParams
 
 	if credentialsAppInstallationID != 0 || credentialsAppID != 0 || credentialsPrivateKeyPath != "" {
@@ -318,6 +326,16 @@ func parseCredentialsUpdateParams() (params.UpdateGithubCredentialsParams, error
 		updateParams.Description = &credentialsDescription
 	}
 
+	// The zero values are meaningful for these two, so only send them if
+	// the flag was explicitly set on the command line.
+	if cmd.Flags().Changed("reserve-usage-enabled") {
+		updateParams.ReserveUsageEnabled = &credentialsReserveEnabled
+	}
+
+	if cmd.Flags().Changed("reserve-usage-percentage") {
+		updateParams.ReserveUsagePercentage = &credentialsReservePercent
+	}
+
 	if credentialsOAuthToken != "" {
 		if updateParams.PAT == nil {
 			updateParams.PAT = &params.GithubPAT{}
@@ -390,6 +408,10 @@ func formatOneGithubCredential(cred params.ForgeCredentials) {
 	t.AppendRow(table.Row{"Upload URL", cred.UploadBaseURL})
 	t.AppendRow(table.Row{"Type", cred.AuthType})
 	t.AppendRow(table.Row{"Endpoint", cred.Endpoint.Name})
+	t.AppendRow(table.Row{"Reserve usage enabled", cred.ReserveUsageEnabled})
+	if cred.ReserveUsageEnabled {
+		t.AppendRow(table.Row{"Reserve usage percentage", fmt.Sprintf("%d%%", cred.ReserveUsagePercentage)})
+	}
 	if resetMinutes > 0 {
 		t.AppendRow(table.Row{"", ""})
 		t.AppendRow(table.Row{"Remaining API requests", cred.RateLimit.Remaining})

database/sql/file_store_test.go

@@ -207,7 +207,7 @@ func (s *FileStoreTestSuite) TestListFileObjects() {
 func (s *FileStoreTestSuite) TestListFileObjectsPagination() {
 	// Create more files to test pagination
 	for i := 0; i < 5; i++ {
-		content := []byte(fmt.Sprintf("File %d", i))
+		content := fmt.Appendf(nil, "File %d", i)
 		param := params.CreateFileObjectParams{
 			Name: fmt.Sprintf("page-test-%d.txt", i),
 			Size: int64(len(content)),
@@ -548,7 +548,7 @@ func (s *FileStoreTestSuite) TestSearchFileObjectByTagsEmptyTags() {
 func (s *FileStoreTestSuite) TestSearchFileObjectByTagsPagination() {
 	// Create multiple files with the same tag
 	for i := 0; i < 5; i++ {
-		content := []byte(fmt.Sprintf("Pagination test file %d", i))
+		content := fmt.Appendf(nil, "Pagination test file %d", i)
 		param := params.CreateFileObjectParams{
 			Name: fmt.Sprintf("page-search-%d.txt", i),
 			Size: int64(len(content)),
@@ -768,7 +768,7 @@ func (s *FileStoreTestSuite) TestSearchFileObjectByTagsOrderByCreatedAt() {
 func (s *FileStoreTestSuite) TestPaginationFieldsLastPage() {
 	// Create exactly 5 files
 	for i := 0; i < 5; i++ {
-		content := []byte(fmt.Sprintf("Last page test %d", i))
+		content := fmt.Appendf(nil, "Last page test %d", i)
 		param := params.CreateFileObjectParams{
 			Name: fmt.Sprintf("last-page-test-%d.txt", i),
 			Size: int64(len(content)),

database/sql/github.go

@@ -274,12 +274,14 @@ func (s *sqlDatabase) CreateGithubCredentials(ctx context.Context, param params.
 		}
 
 		creds = GithubCredentials{
-			Name:         param.Name,
-			Description:  param.Description,
-			EndpointName: &endpoint.Name,
-			AuthType:     param.AuthType,
-			Payload:      data,
-			UserID:       &userID,
+			Name:                   param.Name,
+			Description:            param.Description,
+			EndpointName:           &endpoint.Name,
+			AuthType:               param.AuthType,
+			Payload:                data,
+			UserID:                 &userID,
+			ReserveUsageEnabled:    param.ReserveUsageEnabled,
+			ReserveUsagePercentage: param.ReserveUsagePercentage,
 		}
 
 		if err := tx.Create(&creds).Error; err != nil {
@@ -432,6 +434,12 @@ func (s *sqlDatabase) UpdateGithubCredentials(ctx context.Context, id uint, para
 		if param.Description != nil && *param.Description != creds.Description {
 			updates["description"] = *param.Description
 		}
+		if param.ReserveUsageEnabled != nil && *param.ReserveUsageEnabled != creds.ReserveUsageEnabled {
+			updates["reserve_usage_enabled"] = *param.ReserveUsageEnabled
+		}
+		if param.ReserveUsagePercentage != nil && *param.ReserveUsagePercentage != creds.ReserveUsagePercentage {
+			updates["reserve_usage_percentage"] = *param.ReserveUsagePercentage
+		}
 
 		var data []byte
 		var err error

database/sql/migrations/0007_credentials_reserve_usage.go

@@ -0,0 +1,41 @@
+// Copyright 2026 Cloudbase Solutions SRL
+//
+//    Licensed under the Apache License, Version 2.0 (the "License"); you may
+//    not use this file except in compliance with the License. You may obtain
+//    a copy of the License at
+//
+//         http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+//    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+//    License for the specific language governing permissions and limitations
+//    under the License.
+
+package migrations
+
+import (
+	"github.com/go-gormigrate/gormigrate/v2"
+	"gorm.io/gorm"
+)
+
+// githubCredentials0007 is a minimal stub that only declares the new
+// columns: whether a slice of the credential's rate limit budget is
+// reserved for critical operations (such as runner deletion), and how
+// large that slice is, as a percentage of the hourly limit.
+
+type githubCredentials0007 struct {
+	ReserveUsageEnabled    bool
+	ReserveUsagePercentage int
+}
+
+func (githubCredentials0007) TableName() string { return "github_credentials" }
+
+func init() {
+	Register(&gormigrate.Migration{
+		ID: "0007_credentials_reserve_usage",
+		Migrate: func(tx *gorm.DB) error {
+			return tx.AutoMigrate(&githubCredentials0007{})
+		},
+	})
+}

database/sql/models.go

@@ -565,6 +565,12 @@ type GithubCredentials struct {
 	Endpoint     GithubEndpoint `gorm:"foreignKey:EndpointName"`
 	EndpointName *string        `gorm:"index"`
 
+	// ReserveUsageEnabled toggles whether or not to allocate a certain
+	// percentage of the available rate limit to critical operations such
+	// as delete operations for runners that have finished their jobs.
+	ReserveUsageEnabled    bool
+	ReserveUsagePercentage int
+
 	Repositories  []Repository   `gorm:"foreignKey:CredentialsID"`
 	Organizations []Organization `gorm:"foreignKey:CredentialsID"`
 	Enterprises   []Enterprise   `gorm:"foreignKey:CredentialsID"`

database/sql/util.go

@@ -1139,19 +1139,21 @@ func (s *sqlDatabase) sqlToCommonForgeCredentials(creds GithubCredentials) (para
 	}
 
 	commonCreds := params.ForgeCredentials{
-		ID:                 creds.ID,
-		Name:               creds.Name,
-		Description:        creds.Description,
-		APIBaseURL:         creds.Endpoint.APIBaseURL,
-		BaseURL:            creds.Endpoint.BaseURL,
-		UploadBaseURL:      creds.Endpoint.UploadBaseURL,
-		CABundle:           creds.Endpoint.CACertBundle,
-		AuthType:           creds.AuthType,
-		CreatedAt:          creds.CreatedAt,
-		UpdatedAt:          creds.UpdatedAt,
-		ForgeType:          creds.Endpoint.EndpointType,
-		Endpoint:           ep,
-		CredentialsPayload: data,
+		ID:                     creds.ID,
+		Name:                   creds.Name,
+		Description:            creds.Description,
+		APIBaseURL:             creds.Endpoint.APIBaseURL,
+		BaseURL:                creds.Endpoint.BaseURL,
+		UploadBaseURL:          creds.Endpoint.UploadBaseURL,
+		CABundle:               creds.Endpoint.CACertBundle,
+		AuthType:               creds.AuthType,
+		CreatedAt:              creds.CreatedAt,
+		UpdatedAt:              creds.UpdatedAt,
+		ForgeType:              creds.Endpoint.EndpointType,
+		Endpoint:               ep,
+		CredentialsPayload:     data,
+		ReserveUsageEnabled:    creds.ReserveUsageEnabled,
+		ReserveUsagePercentage: creds.ReserveUsagePercentage,
 	}
 
 	for _, repo := range creds.Repositories {

params/params.go

@@ -1276,6 +1276,16 @@ type ForgeCredentials struct {
 
 	ForgeType EndpointType `json:"forge_type,omitempty"`
 
+	// ReserveUsageEnabled toggles whether or not to allocate a certain
+	// percentage of the available rate limit to critical operations such
+	// as delete operations for runners that have finished their jobs.
+	ReserveUsageEnabled bool `json:"reserve_usage_enabled,omitempty"`
+	// ReserveUsagePercentage is the percentage of available rate limit reserved
+	// for critical operations. Setting this value too high will negatively impact
+	// normal operations, so it is capped at 50%. A value between 5% and 20%
+	// should be safe on most setups. Adjust this based on your usage patterns.
+	ReserveUsagePercentage int `json:"reserve_usage_percentage,omitempty"`
+
 	Repositories  []Repository     `json:"repositories,omitempty"`
 	Organizations []Organization   `json:"organizations,omitempty"`
 	Enterprises   []Enterprise     `json:"enterprises,omitempty"`
@@ -1292,6 +1302,52 @@ func (g ForgeCredentials) GetID() uint {
 	return g.ID
 }
 
+// reserveThreshold returns the number of API calls held in reserve for
+// critical operations (such as removing runners that finished their jobs).
+// Zero when usage reservation is disabled or no rate limit was observed.
+func (g ForgeCredentials) reserveThreshold() int {
+	if !g.ReserveUsageEnabled || g.RateLimit == nil {
+		return 0
+	}
+	return g.RateLimit.Limit * g.ReserveUsagePercentage / 100
+}
+
+// rateLimitReached reports whether the remaining quota has dropped to or
+// below the given threshold, and when the quota resets. The reset time is
+// returned whenever rate limit info was recorded, regardless of the verdict.
+// It is the zero time only when no info is available (Gitea, GHES with rate
+// limiting disabled, or no forge response observed yet). When no rate limits
+// are available, the credentials are considered unlimited. A quota whose
+// reset time has passed is treated as refreshed, even if we have not yet
+// observed a fresh response confirming it, otherwise a controller that
+// stopped making API calls due to the limit would never notice the reset.
+func (g ForgeCredentials) rateLimitReached(threshold int) (bool, time.Time) {
+	if g.RateLimit == nil || g.RateLimit.Limit == 0 {
+		return false, time.Time{}
+	}
+	resetAt := g.RateLimit.ResetAt()
+	if !time.Now().Before(resetAt) {
+		return false, resetAt
+	}
+	return g.RateLimit.Remaining <= threshold, resetAt
+}
+
+// RateLimitReached reports whether these credentials should be considered
+// rate limited for normal (non-critical) operations, and when the quota
+// resets. When usage reservation is enabled, normal operations are
+// considered limited once the remaining quota dips into the reserved
+// percentage, keeping the reserve available for critical operations.
+func (g ForgeCredentials) RateLimitReached() (bool, time.Time) {
+	return g.rateLimitReached(g.reserveThreshold())
+}
+
+// CriticalRateLimitReached reports whether even critical operations (such
+// as removing runners that finished their jobs) should back off, meaning
+// the quota is fully exhausted, and when it resets.
+func (g ForgeCredentials) CriticalRateLimitReached() (bool, time.Time) {
+	return g.rateLimitReached(0)
+}
+
 func (g ForgeCredentials) GetHTTPClient(ctx context.Context) (*http.Client, error) {
 	var roots *x509.CertPool
 	if g.CABundle != nil {