Merge pull request #113485 from MikeSpreitzer/apf-borrowing

Add borrowing between priority levels in APF

api/openapi-spec/v3/apis__flowcontrol.apiserver.k8s.io__v1beta2_openapi.json

@@ -259,6 +259,16 @@
             "format": "int32",
             "type": "integer"
           },
+          "borrowingLimitPercent": {
+            "description": "`borrowingLimitPercent`, if present, configures a limit on how many seats this priority level can borrow from other priority levels. The limit is known as this level's BorrowingConcurrencyLimit (BorrowingCL) and is a limit on the total number of seats that this level may borrow at any one time. This field holds the ratio of that limit to the level's nominal concurrency limit. When this field is non-nil, it must hold a non-negative integer and the limit is calculated as follows.\n\nBorrowingCL(i) = round( NominalCL(i) * borrowingLimitPercent(i)/100.0 )\n\nThe value of this field can be more than 100, implying that this priority level can borrow a number of seats that is greater than its own nominal concurrency limit (NominalCL). When this field is left `nil`, the limit is effectively infinite.",
+            "format": "int32",
+            "type": "integer"
+          },
+          "lendablePercent": {
+            "description": "`lendablePercent` prescribes the fraction of the level's NominalCL that can be borrowed by other priority levels. The value of this field must be between 0 and 100, inclusive, and it defaults to 0. The number of seats that other levels can borrow from this level, known as this level's LendableConcurrencyLimit (LendableCL), is defined as follows.\n\nLendableCL(i) = round( NominalCL(i) * lendablePercent(i)/100.0 )",
+            "format": "int32",
+            "type": "integer"
+          },
           "limitResponse": {
             "allOf": [
               {

api/openapi-spec/v3/apis__flowcontrol.apiserver.k8s.io__v1beta3_openapi.json

@@ -255,6 +255,16 @@
       "io.k8s.api.flowcontrol.v1beta3.LimitedPriorityLevelConfiguration": {
         "description": "LimitedPriorityLevelConfiguration specifies how to handle requests that are subject to limits. It addresses two issues:\n  - How are requests for this priority level limited?\n  - What should be done with requests that exceed the limit?",
         "properties": {
+          "borrowingLimitPercent": {
+            "description": "`borrowingLimitPercent`, if present, configures a limit on how many seats this priority level can borrow from other priority levels. The limit is known as this level's BorrowingConcurrencyLimit (BorrowingCL) and is a limit on the total number of seats that this level may borrow at any one time. This field holds the ratio of that limit to the level's nominal concurrency limit. When this field is non-nil, it must hold a non-negative integer and the limit is calculated as follows.\n\nBorrowingCL(i) = round( NominalCL(i) * borrowingLimitPercent(i)/100.0 )\n\nThe value of this field can be more than 100, implying that this priority level can borrow a number of seats that is greater than its own nominal concurrency limit (NominalCL). When this field is left `nil`, the limit is effectively infinite.",
+            "format": "int32",
+            "type": "integer"
+          },
+          "lendablePercent": {
+            "description": "`lendablePercent` prescribes the fraction of the level's NominalCL that can be borrowed by other priority levels. The value of this field must be between 0 and 100, inclusive, and it defaults to 0. The number of seats that other levels can borrow from this level, known as this level's LendableConcurrencyLimit (LendableCL), is defined as follows.\n\nLendableCL(i) = round( NominalCL(i) * lendablePercent(i)/100.0 )",
+            "format": "int32",
+            "type": "integer"
+          },
           "limitResponse": {
             "allOf": [
               {

pkg/api/testing/defaulting_test.go

@@ -176,6 +176,14 @@ func TestDefaulting(t *testing.T) {
 		{Group: "scheduling.k8s.io", Version: "v1alpha1", Kind: "PriorityClassList"}:                               {},
 		{Group: "scheduling.k8s.io", Version: "v1beta1", Kind: "PriorityClassList"}:                                {},
 		{Group: "scheduling.k8s.io", Version: "v1", Kind: "PriorityClassList"}:                                     {},
+		{Group: "flowcontrol.apiserver.k8s.io", Version: "v1alpha1", Kind: "PriorityLevelConfiguration"}:           {},
+		{Group: "flowcontrol.apiserver.k8s.io", Version: "v1alpha1", Kind: "PriorityLevelConfigurationList"}:       {},
+		{Group: "flowcontrol.apiserver.k8s.io", Version: "v1beta1", Kind: "PriorityLevelConfiguration"}:            {},
+		{Group: "flowcontrol.apiserver.k8s.io", Version: "v1beta1", Kind: "PriorityLevelConfigurationList"}:        {},
+		{Group: "flowcontrol.apiserver.k8s.io", Version: "v1beta2", Kind: "PriorityLevelConfiguration"}:            {},
+		{Group: "flowcontrol.apiserver.k8s.io", Version: "v1beta2", Kind: "PriorityLevelConfigurationList"}:        {},
+		{Group: "flowcontrol.apiserver.k8s.io", Version: "v1beta3", Kind: "PriorityLevelConfiguration"}:            {},
+		{Group: "flowcontrol.apiserver.k8s.io", Version: "v1beta3", Kind: "PriorityLevelConfigurationList"}:        {},
 	}
 
 	f := fuzz.New().NilChance(.5).NumElements(1, 1).RandSource(rand.NewSource(1))

pkg/api/testing/fuzzer.go

@@ -38,6 +38,7 @@ import (
 	corefuzzer "k8s.io/kubernetes/pkg/apis/core/fuzzer"
 	discoveryfuzzer "k8s.io/kubernetes/pkg/apis/discovery/fuzzer"
 	extensionsfuzzer "k8s.io/kubernetes/pkg/apis/extensions/fuzzer"
+	flowcontrolfuzzer "k8s.io/kubernetes/pkg/apis/flowcontrol/fuzzer"
 	networkingfuzzer "k8s.io/kubernetes/pkg/apis/networking/fuzzer"
 	policyfuzzer "k8s.io/kubernetes/pkg/apis/policy/fuzzer"
 	rbacfuzzer "k8s.io/kubernetes/pkg/apis/rbac/fuzzer"
@@ -107,4 +108,5 @@ var FuzzerFuncs = fuzzer.MergeFuzzerFuncs(
 	metafuzzer.Funcs,
 	schedulingfuzzer.Funcs,
 	discoveryfuzzer.Funcs,
+	flowcontrolfuzzer.Funcs,
 )

pkg/apis/flowcontrol/fuzzer/fuzzer.go

@@ -0,0 +1,37 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package fuzzer
+
+import (
+	fuzz "github.com/google/gofuzz"
+
+	runtimeserializer "k8s.io/apimachinery/pkg/runtime/serializer"
+	"k8s.io/kubernetes/pkg/apis/flowcontrol"
+)
+
+// Funcs returns the fuzzer functions for the flowcontrol api group.
+var Funcs = func(codecs runtimeserializer.CodecFactory) []interface{} {
+	return []interface{}{
+		func(obj *flowcontrol.LimitedPriorityLevelConfiguration, c fuzz.Continue) {
+			c.FuzzNoCustom(obj) // fuzz self without calling this function again
+			if obj.LendablePercent == nil {
+				i := int32(0)
+				obj.LendablePercent = &i
+			}
+		},
+	}
+}

pkg/apis/flowcontrol/internalbootstrap/defaults_test.go

@@ -17,31 +17,46 @@ limitations under the License.
 package internalbootstrap
 
 import (
+	"fmt"
 	"testing"
 
+	"github.com/google/go-cmp/cmp"
+
 	flowcontrol "k8s.io/api/flowcontrol/v1beta3"
 	apiequality "k8s.io/apimachinery/pkg/api/equality"
 	"k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap"
 )
 
-func TestMandatoryAlreadyDefaulted(t *testing.T) {
+func TestBootstrapConfigurationWithDefaulted(t *testing.T) {
 	scheme := NewAPFScheme()
-	for _, obj := range bootstrap.MandatoryFlowSchemas {
-		obj2 := obj.DeepCopyObject().(*flowcontrol.FlowSchema)
-		scheme.Default(obj2)
-		if apiequality.Semantic.DeepEqual(obj, obj2) {
-			t.Logf("Defaulting makes no change to %#+v", *obj)
-		} else {
-			t.Errorf("Defaulting changed %#+v to %#+v", *obj, *obj2)
-		}
+
+	bootstrapFlowSchemas := make([]*flowcontrol.FlowSchema, 0)
+	bootstrapFlowSchemas = append(bootstrapFlowSchemas, bootstrap.MandatoryFlowSchemas...)
+	bootstrapFlowSchemas = append(bootstrapFlowSchemas, bootstrap.SuggestedFlowSchemas...)
+	for _, original := range bootstrapFlowSchemas {
+		t.Run(fmt.Sprintf("FlowSchema/%s", original.Name), func(t *testing.T) {
+			defaulted := original.DeepCopyObject().(*flowcontrol.FlowSchema)
+			scheme.Default(defaulted)
+			if apiequality.Semantic.DeepEqual(original, defaulted) {
+				t.Logf("Defaulting makes no change to FlowSchema: %q", original.Name)
+				return
+			}
+			t.Errorf("Expected defaulting to not change FlowSchema: %q, diff: %s", original.Name, cmp.Diff(original, defaulted))
+		})
 	}
-	for _, obj := range bootstrap.MandatoryPriorityLevelConfigurations {
-		obj2 := obj.DeepCopyObject().(*flowcontrol.PriorityLevelConfiguration)
-		scheme.Default(obj2)
-		if apiequality.Semantic.DeepEqual(obj, obj2) {
-			t.Logf("Defaulting makes no change to %#+v", *obj)
-		} else {
-			t.Errorf("Defaulting changed %#+v to %#+v", *obj, *obj2)
-		}
+
+	bootstrapPriorityLevels := make([]*flowcontrol.PriorityLevelConfiguration, 0)
+	bootstrapPriorityLevels = append(bootstrapPriorityLevels, bootstrap.MandatoryPriorityLevelConfigurations...)
+	bootstrapPriorityLevels = append(bootstrapPriorityLevels, bootstrap.SuggestedPriorityLevelConfigurations...)
+	for _, original := range bootstrapPriorityLevels {
+		t.Run(fmt.Sprintf("PriorityLevelConfiguration/%s", original.Name), func(t *testing.T) {
+			defaulted := original.DeepCopyObject().(*flowcontrol.PriorityLevelConfiguration)
+			scheme.Default(defaulted)
+			if apiequality.Semantic.DeepEqual(original, defaulted) {
+				t.Logf("Defaulting makes no change to PriorityLevelConfiguration: %q", original.Name)
+				return
+			}
+			t.Errorf("Expected defaulting to not change PriorityLevelConfiguration: %q, diff: %s", original.Name, cmp.Diff(original, defaulted))
+		})
 	}
 }

pkg/apis/flowcontrol/types.go

@@ -416,6 +416,35 @@ type LimitedPriorityLevelConfiguration struct {
 
 	// `limitResponse` indicates what to do with requests that can not be executed right now
 	LimitResponse LimitResponse
+
+	// `lendablePercent` prescribes the fraction of the level's NominalCL that
+	// can be borrowed by other priority levels. The value of this
+	// field must be between 0 and 100, inclusive, and it defaults to 0.
+	// The number of seats that other levels can borrow from this level, known
+	// as this level's LendableConcurrencyLimit (LendableCL), is defined as follows.
+	//
+	// LendableCL(i) = round( NominalCL(i) * lendablePercent(i)/100.0 )
+	//
+	// +optional
+	LendablePercent *int32
+
+	// `borrowingLimitPercent`, if present, configures a limit on how many
+	// seats this priority level can borrow from other priority levels.
+	// The limit is known as this level's BorrowingConcurrencyLimit
+	// (BorrowingCL) and is a limit on the total number of seats that this
+	// level may borrow at any one time.
+	// This field holds the ratio of that limit to the level's nominal
+	// concurrency limit. When this field is non-nil, it must hold a
+	// non-negative integer and the limit is calculated as follows.
+	//
+	// BorrowingCL(i) = round( NominalCL(i) * borrowingLimitPercent(i)/100.0 )
+	//
+	// The value of this field can be more than 100, implying that this
+	// priority level can borrow a number of seats that is greater than
+	// its own nominal concurrency limit (NominalCL).
+	// When this field is left `nil`, the limit is effectively infinite.
+	// +optional
+	BorrowingLimitPercent *int32
 }
 
 // LimitResponse defines how to handle requests that can not be executed right now.

pkg/apis/flowcontrol/v1alpha1/defaults.go

@@ -44,6 +44,10 @@ func SetDefaults_LimitedPriorityLevelConfiguration(lplc *v1alpha1.LimitedPriorit
 	if lplc.AssuredConcurrencyShares == 0 {
 		lplc.AssuredConcurrencyShares = PriorityLevelConfigurationDefaultAssuredConcurrencyShares
 	}
+	if lplc.LendablePercent == nil {
+		lplc.LendablePercent = new(int32)
+		*lplc.LendablePercent = 0
+	}
 }
 
 // SetDefaults_FlowSchema sets default values for flow schema

pkg/apis/flowcontrol/v1alpha1/defaults_test.go

@@ -0,0 +1,80 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1alpha1
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+
+	flowcontrolv1alpha1 "k8s.io/api/flowcontrol/v1alpha1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/utils/pointer"
+)
+
+func TestDefaultWithPriorityLevelConfiguration(t *testing.T) {
+	tests := []struct {
+		name     string
+		original runtime.Object
+		expected runtime.Object
+	}{
+		{
+			name: "LendablePercent is not specified, should default to zero",
+			original: &flowcontrolv1alpha1.PriorityLevelConfiguration{
+				Spec: flowcontrolv1alpha1.PriorityLevelConfigurationSpec{
+					Type: flowcontrolv1alpha1.PriorityLevelEnablementLimited,
+					Limited: &flowcontrolv1alpha1.LimitedPriorityLevelConfiguration{
+						AssuredConcurrencyShares: 5,
+						LimitResponse: flowcontrolv1alpha1.LimitResponse{
+							Type: flowcontrolv1alpha1.LimitResponseTypeReject,
+						},
+					},
+				},
+			},
+			expected: &flowcontrolv1alpha1.PriorityLevelConfiguration{
+				Spec: flowcontrolv1alpha1.PriorityLevelConfigurationSpec{
+					Type: flowcontrolv1alpha1.PriorityLevelEnablementLimited,
+					Limited: &flowcontrolv1alpha1.LimitedPriorityLevelConfiguration{
+						AssuredConcurrencyShares: 5,
+						LendablePercent:          pointer.Int32(0),
+						LimitResponse: flowcontrolv1alpha1.LimitResponse{
+							Type: flowcontrolv1alpha1.LimitResponseTypeReject,
+						},
+					},
+				},
+			},
+		},
+	}
+
+	scheme := runtime.NewScheme()
+	if err := AddToScheme(scheme); err != nil {
+		t.Fatalf("Failed to add to scheme: %v", err)
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			original := test.original
+			expected := test.expected
+
+			scheme.Default(original)
+			if !reflect.DeepEqual(expected, original) {
+				t.Errorf("Expected defaulting to work - diff: %s", cmp.Diff(expected, original))
+			}
+		})
+	}
+}

pkg/apis/flowcontrol/v1beta1/defaults.go

@@ -44,6 +44,10 @@ func SetDefaults_LimitedPriorityLevelConfiguration(lplc *v1beta1.LimitedPriority
 	if lplc.AssuredConcurrencyShares == 0 {
 		lplc.AssuredConcurrencyShares = PriorityLevelConfigurationDefaultAssuredConcurrencyShares
 	}
+	if lplc.LendablePercent == nil {
+		lplc.LendablePercent = new(int32)
+		*lplc.LendablePercent = 0
+	}
 }
 
 // SetDefaults_FlowSchema sets default values for flow schema