From e4e7b3d9ce00549c2b760627fe432a0f92fa3d91 Mon Sep 17 00:00:00 2001 From: njtran Date: Mon, 4 Dec 2023 18:58:26 -0800 Subject: [PATCH 1/4] chore: remove the remainder of the alpha references --- pkg/apis/v1alpha5/labels.go | 122 +-- pkg/apis/v1alpha5/limits.go | 41 - pkg/apis/v1alpha5/machine.go | 156 ---- pkg/apis/v1alpha5/machine_status.go | 66 -- pkg/apis/v1alpha5/provisioner.go | 149 ---- pkg/apis/v1alpha5/provisioner_defaults.go | 22 - pkg/apis/v1alpha5/provisioner_status.go | 51 -- pkg/apis/v1alpha5/provisioner_validation.go | 339 -------- pkg/apis/v1alpha5/register.go | 33 +- pkg/apis/v1alpha5/suite_test.go | 727 ------------------ pkg/apis/v1alpha5/zz_generated.deepcopy.go | 520 ------------- pkg/apis/v1beta1/labels.go | 2 +- pkg/apis/v1beta1/nodeclaim_validation.go | 4 +- .../nodepool_validation_webhook_test.go | 16 +- pkg/cloudprovider/fake/instancetype.go | 7 +- .../metrics/cloudprovider_test.go | 2 +- pkg/cloudprovider/types.go | 2 +- .../disruption/consolidation_test.go | 30 +- pkg/controllers/metrics/node/controller.go | 12 +- .../metrics/nodepool/controller.go | 2 +- pkg/controllers/metrics/pod/controller.go | 6 +- .../node/termination/controller.go | 4 +- .../node/termination/terminator/terminator.go | 7 +- .../nodeclaim/disruption/controller.go | 4 +- pkg/controllers/nodeclaim/disruption/drift.go | 9 +- .../nodeclaim/disruption/drift_test.go | 8 +- .../nodeclaim/disruption/emptiness.go | 2 +- .../nodeclaim/disruption/expiration.go | 8 +- .../nodeclaim/lifecycle/controller.go | 2 +- .../nodeclaim/lifecycle/initialization.go | 4 +- .../nodeclaim/lifecycle/launch_test.go | 7 +- .../nodeclaim/termination/controller.go | 3 +- .../nodepool/counter/controller.go | 8 +- pkg/controllers/provisioning/provisioner.go | 2 +- pkg/controllers/provisioning/suite_test.go | 8 +- pkg/controllers/state/cluster.go | 113 +-- pkg/controllers/state/informer/nodeclaim.go | 2 +- pkg/controllers/state/statenode.go | 60 +- pkg/controllers/state/suite_test.go | 5 +- .../state/zz_generated.deepcopy.go | 21 - pkg/metrics/constants.go | 7 +- pkg/metrics/metrics.go | 4 +- pkg/operator/controller/suite_test.go | 8 +- pkg/scheduling/requirements.go | 5 +- pkg/test/machine.go | 65 -- pkg/test/nodeclaim.go | 2 +- pkg/test/nodes.go | 17 - pkg/test/provisioner.go | 99 --- pkg/utils/nodeclaim/nodeclaim.go | 4 - pkg/utils/nodeclaim/suite_test.go | 21 +- pkg/utils/nodepool/nodepool.go | 74 -- pkg/utils/nodepool/suite_test.go | 172 ----- pkg/webhooks/webhooks.go | 6 +- 53 files changed, 135 insertions(+), 2935 deletions(-) delete mode 100644 pkg/apis/v1alpha5/limits.go delete mode 100644 pkg/apis/v1alpha5/machine.go delete mode 100644 pkg/apis/v1alpha5/machine_status.go delete mode 100644 pkg/apis/v1alpha5/provisioner.go delete mode 100644 pkg/apis/v1alpha5/provisioner_defaults.go delete mode 100644 pkg/apis/v1alpha5/provisioner_status.go delete mode 100644 pkg/apis/v1alpha5/provisioner_validation.go delete mode 100644 pkg/apis/v1alpha5/suite_test.go delete mode 100644 pkg/apis/v1alpha5/zz_generated.deepcopy.go delete mode 100644 pkg/test/machine.go delete mode 100644 pkg/test/provisioner.go diff --git a/pkg/apis/v1alpha5/labels.go b/pkg/apis/v1alpha5/labels.go index a504c39129..f0a03986df 100644 --- a/pkg/apis/v1alpha5/labels.go +++ b/pkg/apis/v1alpha5/labels.go @@ -14,129 +14,9 @@ limitations under the License. package v1alpha5 -import ( - "fmt" - "strings" - - v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/util/sets" -) - -// Well known labels and resources -const ( - ArchitectureAmd64 = "amd64" - ArchitectureArm64 = "arm64" - CapacityTypeSpot = "spot" - CapacityTypeOnDemand = "on-demand" -) - -// Karpenter specific domains and labels -const ( - ProvisionerNameLabelKey = Group + "/provisioner-name" - LabelNodeInitialized = Group + "/initialized" - LabelNodeRegistered = Group + "/registered" - LabelCapacityType = Group + "/capacity-type" -) - // Karpenter specific annotations +// These are deprecated. These should be removed as part of the migration to v1. const ( DoNotEvictPodAnnotationKey = Group + "/do-not-evict" DoNotConsolidateNodeAnnotationKey = Group + "/do-not-consolidate" - EmptinessTimestampAnnotationKey = Group + "/emptiness-timestamp" - MachineLinkedAnnotationKey = Group + "/linked" - MachineManagedByAnnotationKey = Group + "/managed-by" - ProvisionerHashAnnotationKey = Group + "/provisioner-hash" - - ProviderCompatabilityAnnotationKey = CompatabilityGroup + "/provider" -) - -// Karpenter specific finalizers -const ( - TerminationFinalizer = Group + "/termination" ) - -var ( - // RestrictedLabelDomains are either prohibited by the kubelet or reserved by karpenter - RestrictedLabelDomains = sets.NewString( - "kubernetes.io", - "k8s.io", - Group, - ) - - // LabelDomainExceptions are sub-domains of the RestrictedLabelDomains but allowed because - // they are not used in a context where they may be passed as argument to kubelet. - LabelDomainExceptions = sets.NewString( - "kops.k8s.io", - v1.LabelNamespaceSuffixNode, - v1.LabelNamespaceNodeRestriction, - ) - - // WellKnownLabels are labels that belong to the RestrictedLabelDomains but allowed. - // Karpenter is aware of these labels, and they can be used to further narrow down - // the range of the corresponding values by either provisioner or pods. - WellKnownLabels = sets.New( - ProvisionerNameLabelKey, - v1.LabelTopologyZone, - v1.LabelTopologyRegion, - v1.LabelInstanceTypeStable, - v1.LabelArchStable, - v1.LabelOSStable, - LabelCapacityType, - ) - - // RestrictedLabels are labels that should not be used - // because they may interfere with the internal provisioning logic. - RestrictedLabels = sets.NewString( - EmptinessTimestampAnnotationKey, - v1.LabelHostname, - ) - - // NormalizedLabels translate aliased concepts into the controller's - // WellKnownLabels. Pod requirements are translated for compatibility. - NormalizedLabels = map[string]string{ - v1.LabelFailureDomainBetaZone: v1.LabelTopologyZone, - "beta.kubernetes.io/arch": v1.LabelArchStable, - "beta.kubernetes.io/os": v1.LabelOSStable, - v1.LabelInstanceType: v1.LabelInstanceTypeStable, - v1.LabelFailureDomainBetaRegion: v1.LabelTopologyRegion, - } -) - -// IsRestrictedLabel returns an error if the label is restricted. -func IsRestrictedLabel(key string) error { - if WellKnownLabels.Has(key) { - return nil - } - if IsRestrictedNodeLabel(key) { - return fmt.Errorf("label %s is restricted; specify a well known label: %v, or a custom label that does not use a restricted domain: %v", key, sets.List(WellKnownLabels), RestrictedLabelDomains.List()) - } - return nil -} - -// IsRestrictedNodeLabel returns true if a node label should not be injected by Karpenter. -// They are either known labels that will be injected by cloud providers, -// or label domain managed by other software (e.g., kops.k8s.io managed by kOps). -func IsRestrictedNodeLabel(key string) bool { - if WellKnownLabels.Has(key) { - return true - } - labelDomain := GetLabelDomain(key) - for exceptionLabelDomain := range LabelDomainExceptions { - if strings.HasSuffix(labelDomain, exceptionLabelDomain) { - return false - } - } - for restrictedLabelDomain := range RestrictedLabelDomains { - if strings.HasSuffix(labelDomain, restrictedLabelDomain) { - return true - } - } - return RestrictedLabels.Has(key) -} - -func GetLabelDomain(key string) string { - if parts := strings.SplitN(key, "/", 2); len(parts) == 2 { - return parts[0] - } - return "" -} diff --git a/pkg/apis/v1alpha5/limits.go b/pkg/apis/v1alpha5/limits.go deleted file mode 100644 index 4dcee02170..0000000000 --- a/pkg/apis/v1alpha5/limits.go +++ /dev/null @@ -1,41 +0,0 @@ -/* -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package v1alpha5 - -import ( - "fmt" - - v1 "k8s.io/api/core/v1" -) - -// Limits define bounds on the resources being provisioned by Karpenter -type Limits struct { - // Resources contains all the allocatable resources that Karpenter supports for limiting. - Resources v1.ResourceList `json:"resources,omitempty"` -} - -func (l *Limits) ExceededBy(resources v1.ResourceList) error { - if l == nil || l.Resources == nil { - return nil - } - for resourceName, usage := range resources { - if limit, ok := l.Resources[resourceName]; ok { - if usage.Cmp(limit) > 0 { - return fmt.Errorf("%s resource usage of %v exceeds limit of %v", resourceName, usage.AsDec(), limit.AsDec()) - } - } - } - return nil -} diff --git a/pkg/apis/v1alpha5/machine.go b/pkg/apis/v1alpha5/machine.go deleted file mode 100644 index f5c787d4c8..0000000000 --- a/pkg/apis/v1alpha5/machine.go +++ /dev/null @@ -1,156 +0,0 @@ -/* -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package v1alpha5 - -import ( - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -// MachineSpec describes the desired state of the Machine -type MachineSpec struct { - // Taints will be applied to the machine's node. - // +optional - Taints []v1.Taint `json:"taints,omitempty"` - // StartupTaints are taints that are applied to nodes upon startup which are expected to be removed automatically - // within a short period of time, typically by a DaemonSet that tolerates the taint. These are commonly used by - // daemonsets to allow initialization and enforce startup ordering. StartupTaints are ignored for provisioning - // purposes in that pods are not required to tolerate a StartupTaint in order to have nodes provisioned for them. - // +optional - StartupTaints []v1.Taint `json:"startupTaints,omitempty"` - // Requirements are layered with Labels and applied to every node. - Requirements []v1.NodeSelectorRequirement `json:"requirements,omitempty"` - // Resources models the resource requirements for the Machine to launch - Resources ResourceRequirements `json:"resources,omitempty"` - // Kubelet are options passed to the kubelet when provisioning nodes - // +optional - Kubelet *KubeletConfiguration `json:"kubelet,omitempty"` - // MachineTemplateRef is a reference to an object that defines provider specific configuration - MachineTemplateRef *MachineTemplateRef `json:"machineTemplateRef,omitempty"` -} - -// KubeletConfiguration defines args to be used when configuring kubelet on provisioned nodes. -// They are a subset of the upstream types, recognizing not all options may be supported. -// Wherever possible, the types and names should reflect the upstream kubelet types. -// https://pkg.go.dev/k8s.io/kubelet/config/v1beta1#KubeletConfiguration -// https://github.com/kubernetes/kubernetes/blob/9f82d81e55cafdedab619ea25cabf5d42736dacf/cmd/kubelet/app/options/options.go#L53 -type KubeletConfiguration struct { - // clusterDNS is a list of IP addresses for the cluster DNS server. - // Note that not all providers may use all addresses. - //+optional - ClusterDNS []string `json:"clusterDNS,omitempty"` - // ContainerRuntime is the container runtime to be used with your worker nodes. - // +optional - ContainerRuntime *string `json:"containerRuntime,omitempty"` - // MaxPods is an override for the maximum number of pods that can run on - // a worker node instance. - // +kubebuilder:validation:Minimum:=0 - // +optional - MaxPods *int32 `json:"maxPods,omitempty"` - // PodsPerCore is an override for the number of pods that can run on a worker node - // instance based on the number of cpu cores. This value cannot exceed MaxPods, so, if - // MaxPods is a lower value, that value will be used. - // +kubebuilder:validation:Minimum:=0 - // +optional - PodsPerCore *int32 `json:"podsPerCore,omitempty"` - // SystemReserved contains resources reserved for OS system daemons and kernel memory. - // +optional - SystemReserved v1.ResourceList `json:"systemReserved,omitempty"` - // KubeReserved contains resources reserved for Kubernetes system components. - // +optional - KubeReserved v1.ResourceList `json:"kubeReserved,omitempty"` - // EvictionHard is the map of signal names to quantities that define hard eviction thresholds - // +optional - EvictionHard map[string]string `json:"evictionHard,omitempty"` - // EvictionSoft is the map of signal names to quantities that define soft eviction thresholds - // +optional - EvictionSoft map[string]string `json:"evictionSoft,omitempty"` - // EvictionSoftGracePeriod is the map of signal names to quantities that define grace periods for each eviction signal - // +optional - EvictionSoftGracePeriod map[string]metav1.Duration `json:"evictionSoftGracePeriod,omitempty"` - // EvictionMaxPodGracePeriod is the maximum allowed grace period (in seconds) to use when terminating pods in - // response to soft eviction thresholds being met. - // +optional - EvictionMaxPodGracePeriod *int32 `json:"evictionMaxPodGracePeriod,omitempty"` - // ImageGCHighThresholdPercent is the percent of disk usage after which image - // garbage collection is always run. The percent is calculated by dividing this - // field value by 100, so this field must be between 0 and 100, inclusive. - // When specified, the value must be greater than ImageGCLowThresholdPercent. - // +kubebuilder:validation:Minimum:=0 - // +kubebuilder:validation:Maximum:=100 - // +optional - ImageGCHighThresholdPercent *int32 `json:"imageGCHighThresholdPercent,omitempty"` - // ImageGCLowThresholdPercent is the percent of disk usage before which image - // garbage collection is never run. Lowest disk usage to garbage collect to. - // The percent is calculated by dividing this field value by 100, - // so the field value must be between 0 and 100, inclusive. - // When specified, the value must be less than imageGCHighThresholdPercent - // +kubebuilder:validation:Minimum:=0 - // +kubebuilder:validation:Maximum:=100 - // +optional - ImageGCLowThresholdPercent *int32 `json:"imageGCLowThresholdPercent,omitempty"` - // CPUCFSQuota enables CPU CFS quota enforcement for containers that specify CPU limits. - // +optional - CPUCFSQuota *bool `json:"cpuCFSQuota,omitempty"` -} - -type MachineTemplateRef struct { - // Kind of the referent; More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds" - Kind string `json:"kind,omitempty"` - // Name of the referent; More info: http://kubernetes.io/docs/user-guide/identifiers#names - // +required - Name string `json:"name"` - // API version of the referent - // +optional - APIVersion string `json:"apiVersion,omitempty"` -} - -// ResourceRequirements models the required resources for the Machine to launch -// Ths will eventually be transformed into v1.ResourceRequirements when we support resources.limits -type ResourceRequirements struct { - // Requests describes the minimum required resources for the Machine to launch - // +optional - Requests v1.ResourceList `json:"requests,omitempty"` -} - -// TODO @joinnis: Mark this version as deprecated when v1beta1 APIs are formally released - -// Machine is the Schema for the Machines API -// +kubebuilder:object:root=true -// +kubebuilder:resource:path=machines,scope=Cluster,categories=karpenter -// +kubebuilder:subresource:status -// +kubebuilder:printcolumn:name="Type",type="string",JSONPath=".metadata.labels.node\\.kubernetes\\.io/instance-type",description="" -// +kubebuilder:printcolumn:name="Zone",type="string",JSONPath=".metadata.labels.topology\\.kubernetes\\.io/zone",description="" -// +kubebuilder:printcolumn:name="Node",type="string",JSONPath=".status.nodeName",description="" -// +kubebuilder:printcolumn:name="Ready",type="string",JSONPath=".status.conditions[?(@.type==\"Ready\")].status",description="" -// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp",description="" -// +kubebuilder:printcolumn:name="Capacity",type="string",JSONPath=".metadata.labels.karpenter\\.sh/capacity-type",priority=1,description="" -// +kubebuilder:printcolumn:name="Provisioner",type="string",JSONPath=".metadata.labels.karpenter\\.sh/provisioner-name",priority=1,description="" -// +kubebuilder:printcolumn:name="Template",type="string",JSONPath=".spec.machineTemplateRef.name",priority=1,description="" -type Machine struct { - metav1.TypeMeta `json:",inline"` - metav1.ObjectMeta `json:"metadata,omitempty"` - - Spec MachineSpec `json:"spec,omitempty"` - Status MachineStatus `json:"status,omitempty"` -} - -// MachineList contains a list of Provisioner -// +kubebuilder:object:root=true -type MachineList struct { - metav1.TypeMeta `json:",inline"` - metav1.ListMeta `json:"metadata,omitempty"` - Items []Machine `json:"items"` -} diff --git a/pkg/apis/v1alpha5/machine_status.go b/pkg/apis/v1alpha5/machine_status.go deleted file mode 100644 index cb699d8bae..0000000000 --- a/pkg/apis/v1alpha5/machine_status.go +++ /dev/null @@ -1,66 +0,0 @@ -/* -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package v1alpha5 - -import ( - v1 "k8s.io/api/core/v1" - "knative.dev/pkg/apis" -) - -// MachineStatus defines the observed state of Machine -type MachineStatus struct { - // NodeName is the name of the corresponding node object - // +optional - NodeName string `json:"nodeName,omitempty"` - // ProviderID of the corresponding node object - // +optional - ProviderID string `json:"providerID,omitempty"` - // Capacity is the estimated full capacity of the machine - // +optional - Capacity v1.ResourceList `json:"capacity,omitempty"` - // Allocatable is the estimated allocatable capacity of the machine - // +optional - Allocatable v1.ResourceList `json:"allocatable,omitempty"` - // Conditions contains signals for health and readiness - // +optional - Conditions apis.Conditions `json:"conditions,omitempty"` -} - -func (in *Machine) StatusConditions() apis.ConditionManager { - return apis.NewLivingConditionSet(LivingConditions...).Manage(in) -} - -var LivingConditions = []apis.ConditionType{ - MachineLaunched, - MachineRegistered, - MachineInitialized, -} - -var ( - MachineLaunched apis.ConditionType = "MachineLaunched" - MachineRegistered apis.ConditionType = "MachineRegistered" - MachineInitialized apis.ConditionType = "MachineInitialized" - MachineDrifted apis.ConditionType = "MachineDrifted" - MachineEmpty apis.ConditionType = "MachineEmpty" - MachineExpired apis.ConditionType = "MachineExpired" -) - -func (in *Machine) GetConditions() apis.Conditions { - return in.Status.Conditions -} - -func (in *Machine) SetConditions(conditions apis.Conditions) { - in.Status.Conditions = conditions -} diff --git a/pkg/apis/v1alpha5/provisioner.go b/pkg/apis/v1alpha5/provisioner.go deleted file mode 100644 index 507494b13d..0000000000 --- a/pkg/apis/v1alpha5/provisioner.go +++ /dev/null @@ -1,149 +0,0 @@ -/* -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package v1alpha5 - -import ( - "encoding/json" - "fmt" - "sort" - - "github.com/mitchellh/hashstructure/v2" - "github.com/samber/lo" - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "knative.dev/pkg/ptr" -) - -// ProvisionerSpec is the top level provisioner specification. Provisioners -// launch nodes in response to pods that are unschedulable. A single provisioner -// is capable of managing a diverse set of nodes. Node properties are determined -// from a combination of provisioner and pod scheduling constraints. -type ProvisionerSpec struct { - // Annotations are applied to every node. - //+optional - Annotations map[string]string `json:"annotations,omitempty"` - // Labels are layered with Requirements and applied to every node. - //+optional - Labels map[string]string `json:"labels,omitempty"` - // Taints will be applied to every node launched by the Provisioner. If - // specified, the provisioner will not provision nodes for pods that do not - // have matching tolerations. Additional taints will be created that match - // pod tolerations on a per-node basis. - // +optional - Taints []v1.Taint `json:"taints,omitempty"` - // StartupTaints are taints that are applied to nodes upon startup which are expected to be removed automatically - // within a short period of time, typically by a DaemonSet that tolerates the taint. These are commonly used by - // daemonsets to allow initialization and enforce startup ordering. StartupTaints are ignored for provisioning - // purposes in that pods are not required to tolerate a StartupTaint in order to have nodes provisioned for them. - // +optional - StartupTaints []v1.Taint `json:"startupTaints,omitempty"` - // Requirements are layered with Labels and applied to every node. - Requirements []v1.NodeSelectorRequirement `json:"requirements,omitempty" hash:"ignore"` - // KubeletConfiguration are options passed to the kubelet when provisioning nodes - //+optional - KubeletConfiguration *KubeletConfiguration `json:"kubeletConfiguration,omitempty"` - // Provider contains fields specific to your cloudprovider. - // +kubebuilder:pruning:PreserveUnknownFields - Provider *Provider `json:"provider,omitempty" hash:"ignore"` - // ProviderRef is a reference to a dedicated CRD for the chosen provider, that holds - // additional configuration options - // +optional - ProviderRef *MachineTemplateRef `json:"providerRef,omitempty" hash:"ignore"` - // TTLSecondsAfterEmpty is the number of seconds the controller will wait - // before attempting to delete a node, measured from when the node is - // detected to be empty. A Node is considered to be empty when it does not - // have pods scheduled to it, excluding daemonsets. - // - // Termination due to no utilization is disabled if this field is not set. - // +optional - TTLSecondsAfterEmpty *int64 `json:"ttlSecondsAfterEmpty,omitempty" hash:"ignore"` - // TTLSecondsUntilExpired is the number of seconds the controller will wait - // before terminating a node, measured from when the node is created. This - // is useful to implement features like eventually consistent node upgrade, - // memory leak protection, and disruption testing. - // - // Termination due to expiration is disabled if this field is not set. - // +optional - TTLSecondsUntilExpired *int64 `json:"ttlSecondsUntilExpired,omitempty" hash:"ignore"` - // Limits define a set of bounds for provisioning capacity. - Limits *Limits `json:"limits,omitempty" hash:"ignore"` - // Weight is the priority given to the provisioner during scheduling. A higher - // numerical weight indicates that this provisioner will be ordered - // ahead of other provisioners with lower weights. A provisioner with no weight - // will be treated as if it is a provisioner with a weight of 0. - // +kubebuilder:validation:Minimum:=1 - // +kubebuilder:validation:Maximum:=100 - // +optional - Weight *int32 `json:"weight,omitempty" hash:"ignore"` - // Consolidation are the consolidation parameters - // +optional - Consolidation *Consolidation `json:"consolidation,omitempty" hash:"ignore"` -} - -func (p *Provisioner) Hash() string { - hash, _ := hashstructure.Hash(p.Spec, hashstructure.FormatV2, &hashstructure.HashOptions{ - SlicesAsSets: true, - IgnoreZeroValue: true, - ZeroNil: true, - }) - return fmt.Sprint(hash) -} - -type Consolidation struct { - // Enabled enables consolidation if it has been set - Enabled *bool `json:"enabled,omitempty"` -} - -// +kubebuilder:object:generate=false -type Provider = runtime.RawExtension - -func ProviderAnnotation(p *Provider) map[string]string { - if p == nil { - return nil - } - raw := lo.Must(json.Marshal(p)) // Provider should already have been validated so this shouldn't fail - return map[string]string{ProviderCompatabilityAnnotationKey: string(raw)} -} - -// Provisioner is the Schema for the Provisioners API -// +kubebuilder:object:root=true -// +kubebuilder:resource:path=provisioners,scope=Cluster,categories=karpenter -// +kubebuilder:subresource:status -// +kubebuilder:printcolumn:name="Template",type="string",JSONPath=".spec.providerRef.name",description="" -// +kubebuilder:printcolumn:name="Weight",type="string",JSONPath=".spec.weight",priority=1,description="" -type Provisioner struct { - metav1.TypeMeta `json:",inline"` - metav1.ObjectMeta `json:"metadata,omitempty"` - - Spec ProvisionerSpec `json:"spec,omitempty"` - Status ProvisionerStatus `json:"status,omitempty"` -} - -// ProvisionerList contains a list of Provisioner -// +kubebuilder:object:root=true -type ProvisionerList struct { - metav1.TypeMeta `json:",inline"` - metav1.ListMeta `json:"metadata,omitempty"` - Items []Provisioner `json:"items"` -} - -// OrderByWeight orders the provisioners in the ProvisionerList -// by their priority weight in-place -func (pl *ProvisionerList) OrderByWeight() { - sort.Slice(pl.Items, func(a, b int) bool { - return ptr.Int32Value(pl.Items[a].Spec.Weight) > ptr.Int32Value(pl.Items[b].Spec.Weight) - }) -} diff --git a/pkg/apis/v1alpha5/provisioner_defaults.go b/pkg/apis/v1alpha5/provisioner_defaults.go deleted file mode 100644 index 07e6ef4d76..0000000000 --- a/pkg/apis/v1alpha5/provisioner_defaults.go +++ /dev/null @@ -1,22 +0,0 @@ -/* -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package v1alpha5 - -import ( - "context" -) - -// SetDefaults for the provisioner -func (p *Provisioner) SetDefaults(_ context.Context) {} diff --git a/pkg/apis/v1alpha5/provisioner_status.go b/pkg/apis/v1alpha5/provisioner_status.go deleted file mode 100644 index 1f16fff525..0000000000 --- a/pkg/apis/v1alpha5/provisioner_status.go +++ /dev/null @@ -1,51 +0,0 @@ -/* -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package v1alpha5 - -import ( - v1 "k8s.io/api/core/v1" - "knative.dev/pkg/apis" -) - -// ProvisionerStatus defines the observed state of Provisioner -type ProvisionerStatus struct { - // LastScaleTime is the last time the Provisioner scaled the number - // of nodes - // +optional - // +kubebuilder:validation:Format="date-time" - LastScaleTime *apis.VolatileTime `json:"lastScaleTime,omitempty"` - - // Conditions is the set of conditions required for this provisioner to scale - // its target, and indicates whether or not those conditions are met. - // +optional - Conditions apis.Conditions `json:"conditions,omitempty"` - - // Resources is the list of resources that have been provisioned. - Resources v1.ResourceList `json:"resources,omitempty"` -} - -func (p *Provisioner) StatusConditions() apis.ConditionManager { - return apis.NewLivingConditionSet( - Active, - ).Manage(p) -} - -func (p *Provisioner) GetConditions() apis.Conditions { - return p.Status.Conditions -} - -func (p *Provisioner) SetConditions(conditions apis.Conditions) { - p.Status.Conditions = conditions -} diff --git a/pkg/apis/v1alpha5/provisioner_validation.go b/pkg/apis/v1alpha5/provisioner_validation.go deleted file mode 100644 index e1ac3a357b..0000000000 --- a/pkg/apis/v1alpha5/provisioner_validation.go +++ /dev/null @@ -1,339 +0,0 @@ -/* -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package v1alpha5 - -import ( - "context" - "fmt" - "strconv" - "strings" - - "github.com/samber/lo" - "go.uber.org/multierr" - admissionregistrationv1 "k8s.io/api/admissionregistration/v1" - v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" - "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/apimachinery/pkg/util/validation" - "knative.dev/pkg/apis" - "knative.dev/pkg/ptr" -) - -var ( - SupportedNodeSelectorOps = sets.NewString( - string(v1.NodeSelectorOpIn), - string(v1.NodeSelectorOpNotIn), - string(v1.NodeSelectorOpGt), - string(v1.NodeSelectorOpLt), - string(v1.NodeSelectorOpExists), - string(v1.NodeSelectorOpDoesNotExist), - ) - - SupportedReservedResources = sets.NewString( - v1.ResourceCPU.String(), - v1.ResourceMemory.String(), - v1.ResourceEphemeralStorage.String(), - "pid", - ) - - SupportedEvictionSignals = sets.NewString( - "memory.available", - "nodefs.available", - "nodefs.inodesFree", - "imagefs.available", - "imagefs.inodesFree", - "pid.available", - ) -) - -const ( - providerPath = "provider" - providerRefPath = "providerRef" -) - -func (p *Provisioner) SupportedVerbs() []admissionregistrationv1.OperationType { - return []admissionregistrationv1.OperationType{ - admissionregistrationv1.Create, - admissionregistrationv1.Update, - } -} - -func (p *Provisioner) Validate(ctx context.Context) (errs *apis.FieldError) { - return errs.Also( - apis.ValidateObjectMetadata(p).ViaField("metadata"), - p.Spec.validate(ctx).ViaField("spec"), - ) -} - -func (s *ProvisionerSpec) validate(ctx context.Context) (errs *apis.FieldError) { - return errs.Also( - s.validateTTLSecondsUntilExpired(), - s.validateTTLSecondsAfterEmpty(), - s.Validate(ctx), - ) -} - -func (s *ProvisionerSpec) validateTTLSecondsUntilExpired() (errs *apis.FieldError) { - if ptr.Int64Value(s.TTLSecondsUntilExpired) < 0 { - return errs.Also(apis.ErrInvalidValue("cannot be negative", "ttlSecondsUntilExpired")) - } - return errs -} - -func (s *ProvisionerSpec) validateTTLSecondsAfterEmpty() (errs *apis.FieldError) { - if ptr.Int64Value(s.TTLSecondsAfterEmpty) < 0 { - return errs.Also(apis.ErrInvalidValue("cannot be negative", "ttlSecondsAfterEmpty")) - } - // TTLSecondsAfterEmpty and consolidation are mutually exclusive - if s.Consolidation != nil && ptr.BoolValue(s.Consolidation.Enabled) && s.TTLSecondsAfterEmpty != nil { - return errs.Also(apis.ErrMultipleOneOf("ttlSecondsAfterEmpty", "consolidation.enabled")) - } - return errs -} - -// Validate the constraints -func (s *ProvisionerSpec) Validate(_ context.Context) (errs *apis.FieldError) { - return errs.Also( - s.validateProvider(), - s.validateLabels(), - s.validateTaints(), - s.validateRequirements(), - s.validateKubeletConfiguration().ViaField("kubeletConfiguration"), - ) -} - -func (s *ProvisionerSpec) validateLabels() (errs *apis.FieldError) { - for key, value := range s.Labels { - if key == ProvisionerNameLabelKey { - errs = errs.Also(apis.ErrInvalidKeyName(key, "labels", "restricted")) - } - for _, err := range validation.IsQualifiedName(key) { - errs = errs.Also(apis.ErrInvalidKeyName(key, "labels", err)) - } - for _, err := range validation.IsValidLabelValue(value) { - errs = errs.Also(apis.ErrInvalidValue(fmt.Sprintf("%s, %s", value, err), fmt.Sprintf("labels[%s]", key))) - } - if err := IsRestrictedLabel(key); err != nil { - errs = errs.Also(apis.ErrInvalidKeyName(key, "labels", err.Error())) - } - } - return errs -} - -type taintKeyEffect struct { - Key string - Effect v1.TaintEffect -} - -func (s *ProvisionerSpec) validateTaints() (errs *apis.FieldError) { - existing := map[taintKeyEffect]struct{}{} - errs = errs.Also(s.validateTaintsField(s.Taints, existing, "taints")) - errs = errs.Also(s.validateTaintsField(s.StartupTaints, existing, "startupTaints")) - return errs -} - -func (s *ProvisionerSpec) validateTaintsField(taints []v1.Taint, existing map[taintKeyEffect]struct{}, fieldName string) *apis.FieldError { - var errs *apis.FieldError - for i, taint := range taints { - // Validate Key - if len(taint.Key) == 0 { - errs = errs.Also(apis.ErrInvalidArrayValue(errs, fieldName, i)) - } - for _, err := range validation.IsQualifiedName(taint.Key) { - errs = errs.Also(apis.ErrInvalidArrayValue(err, fieldName, i)) - } - // Validate Value - if len(taint.Value) != 0 { - for _, err := range validation.IsQualifiedName(taint.Value) { - errs = errs.Also(apis.ErrInvalidArrayValue(err, fieldName, i)) - } - } - // Validate effect - switch taint.Effect { - case v1.TaintEffectNoSchedule, v1.TaintEffectPreferNoSchedule, v1.TaintEffectNoExecute, "": - default: - errs = errs.Also(apis.ErrInvalidArrayValue(taint.Effect, "effect", i)) - } - - // Check for duplicate Key/Effect pairs - key := taintKeyEffect{Key: taint.Key, Effect: taint.Effect} - if _, ok := existing[key]; ok { - errs = errs.Also(apis.ErrGeneric(fmt.Sprintf("duplicate taint Key/Effect pair %s=%s", taint.Key, taint.Effect), apis.CurrentField). - ViaFieldIndex("taints", i)) - } - existing[key] = struct{}{} - } - return errs -} - -// This function is used by the provisioner validation webhook to verify the provisioner requirements. -// When this function is called, the provisioner's requirments do not include the requirements from labels. -// Provisioner requirements only support well known labels. -func (s *ProvisionerSpec) validateRequirements() (errs *apis.FieldError) { - for i, requirement := range s.Requirements { - if requirement.Key == ProvisionerNameLabelKey { - errs = errs.Also(apis.ErrInvalidArrayValue(fmt.Sprintf("%s is restricted", requirement.Key), "requirements", i)) - } - if err := ValidateRequirement(requirement); err != nil { - errs = errs.Also(apis.ErrInvalidArrayValue(err, "requirements", i)) - } - } - return errs -} - -// validateProvider checks if exactly one of provider and providerRef are set -func (s *ProvisionerSpec) validateProvider() *apis.FieldError { - if s.Provider != nil && s.ProviderRef != nil { - return apis.ErrMultipleOneOf(providerPath, providerRefPath) - } - if s.Provider == nil && s.ProviderRef == nil { - return apis.ErrMissingOneOf(providerPath, providerRefPath) - } - return nil -} - -func (s *ProvisionerSpec) validateKubeletConfiguration() (errs *apis.FieldError) { - if s.KubeletConfiguration == nil { - return - } - return errs.Also( - validateEvictionThresholds(s.KubeletConfiguration.EvictionHard, "evictionHard"), - validateEvictionThresholds(s.KubeletConfiguration.EvictionSoft, "evictionSoft"), - validateReservedResources(s.KubeletConfiguration.KubeReserved, "kubeReserved"), - validateReservedResources(s.KubeletConfiguration.SystemReserved, "systemReserved"), - s.KubeletConfiguration.validateImageGCHighThresholdPercent(), - s.KubeletConfiguration.validateImageGCLowThresholdPercent(), - s.KubeletConfiguration.validateEvictionSoftGracePeriod(), - s.KubeletConfiguration.validateEvictionSoftPairs(), - ) -} - -func (kc *KubeletConfiguration) validateEvictionSoftGracePeriod() (errs *apis.FieldError) { - for k := range kc.EvictionSoftGracePeriod { - if !SupportedEvictionSignals.Has(k) { - errs = errs.Also(apis.ErrInvalidKeyName(k, "evictionSoftGracePeriod")) - } - } - return errs -} - -func (kc *KubeletConfiguration) validateEvictionSoftPairs() (errs *apis.FieldError) { - evictionSoftKeys := sets.NewString(lo.Keys(kc.EvictionSoft)...) - evictionSoftGracePeriodKeys := sets.NewString(lo.Keys(kc.EvictionSoftGracePeriod)...) - - evictionSoftDiff := evictionSoftKeys.Difference(evictionSoftGracePeriodKeys) - for k := range evictionSoftDiff { - errs = errs.Also(apis.ErrInvalidKeyName(k, "evictionSoft", "Key does not have a matching evictionSoftGracePeriod")) - } - evictionSoftGracePeriodDiff := evictionSoftGracePeriodKeys.Difference(evictionSoftKeys) - for k := range evictionSoftGracePeriodDiff { - errs = errs.Also(apis.ErrInvalidKeyName(k, "evictionSoftGracePeriod", "Key does not have a matching evictionSoft threshold value")) - } - return errs -} - -func validateReservedResources(m v1.ResourceList, fieldName string) (errs *apis.FieldError) { - for k, v := range m { - if !SupportedReservedResources.Has(k.String()) { - errs = errs.Also(apis.ErrInvalidKeyName(k.String(), fieldName)) - } - if v.Value() < 0 { - errs = errs.Also(apis.ErrInvalidValue(v.String(), fmt.Sprintf(`%s["%s"]`, fieldName, k), "Value cannot be a negative resource quantity")) - } - } - return errs -} - -func validateEvictionThresholds(m map[string]string, fieldName string) (errs *apis.FieldError) { - if m == nil { - return - } - for k, v := range m { - if !SupportedEvictionSignals.Has(k) { - errs = errs.Also(apis.ErrInvalidKeyName(k, fieldName)) - } - if strings.HasSuffix(v, "%") { - p, err := strconv.ParseFloat(strings.Trim(v, "%"), 64) - if err != nil { - errs = errs.Also(apis.ErrInvalidValue(v, fmt.Sprintf(`%s["%s"]`, fieldName, k), fmt.Sprintf("Value could not be parsed as a percentage value, %v", err.Error()))) - } - if p < 0 { - errs = errs.Also(apis.ErrInvalidValue(v, fmt.Sprintf(`%s["%s"]`, fieldName, k), "Percentage values cannot be negative")) - } - if p > 100 { - errs = errs.Also(apis.ErrInvalidValue(v, fmt.Sprintf(`%s["%s"]`, fieldName, k), "Percentage values cannot be greater than 100")) - } - } else { - _, err := resource.ParseQuantity(v) - if err != nil { - errs = errs.Also(apis.ErrInvalidValue(v, fmt.Sprintf("%s[%s]", fieldName, k), fmt.Sprintf("Value could not be parsed as a resource quantity, %v", err.Error()))) - } - } - } - return errs -} - -func ValidateRequirement(requirement v1.NodeSelectorRequirement) error { //nolint:gocyclo - var errs error - if normalized, ok := NormalizedLabels[requirement.Key]; ok { - requirement.Key = normalized - } - if !SupportedNodeSelectorOps.Has(string(requirement.Operator)) { - errs = multierr.Append(errs, fmt.Errorf("key %s has an unsupported operator %s not in %s", requirement.Key, requirement.Operator, SupportedNodeSelectorOps.UnsortedList())) - } - if e := IsRestrictedLabel(requirement.Key); e != nil { - errs = multierr.Append(errs, e) - } - for _, err := range validation.IsQualifiedName(requirement.Key) { - errs = multierr.Append(errs, fmt.Errorf("key %s is not a qualified name, %s", requirement.Key, err)) - } - for _, value := range requirement.Values { - for _, err := range validation.IsValidLabelValue(value) { - errs = multierr.Append(errs, fmt.Errorf("invalid value %s for key %s, %s", value, requirement.Key, err)) - } - } - if requirement.Operator == v1.NodeSelectorOpIn && len(requirement.Values) == 0 { - errs = multierr.Append(errs, fmt.Errorf("key %s with operator %s must have a value defined", requirement.Key, requirement.Operator)) - } - if requirement.Operator == v1.NodeSelectorOpGt || requirement.Operator == v1.NodeSelectorOpLt { - if len(requirement.Values) != 1 { - errs = multierr.Append(errs, fmt.Errorf("key %s with operator %s must have a single positive integer value", requirement.Key, requirement.Operator)) - } else { - value, err := strconv.Atoi(requirement.Values[0]) - if err != nil || value < 0 { - errs = multierr.Append(errs, fmt.Errorf("key %s with operator %s must have a single positive integer value", requirement.Key, requirement.Operator)) - } - } - } - return errs -} - -// Validate validateImageGCHighThresholdPercent -func (kc *KubeletConfiguration) validateImageGCHighThresholdPercent() (errs *apis.FieldError) { - if kc.ImageGCHighThresholdPercent != nil && ptr.Int32Value(kc.ImageGCHighThresholdPercent) < ptr.Int32Value(kc.ImageGCLowThresholdPercent) { - return errs.Also(apis.ErrInvalidValue("must be greater than imageGCLowThresholdPercent", "imageGCHighThresholdPercent")) - } - - return errs -} - -// Validate imageGCLowThresholdPercent -func (kc *KubeletConfiguration) validateImageGCLowThresholdPercent() (errs *apis.FieldError) { - if kc.ImageGCHighThresholdPercent != nil && ptr.Int32Value(kc.ImageGCLowThresholdPercent) > ptr.Int32Value(kc.ImageGCHighThresholdPercent) { - return errs.Also(apis.ErrInvalidValue("must be less than imageGCHighThresholdPercent", "imageGCLowThresholdPercent")) - } - - return errs -} diff --git a/pkg/apis/v1alpha5/register.go b/pkg/apis/v1alpha5/register.go index d92bfa4615..91d9c1aa1c 100644 --- a/pkg/apis/v1alpha5/register.go +++ b/pkg/apis/v1alpha5/register.go @@ -14,37 +14,6 @@ limitations under the License. package v1alpha5 -import ( - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" - "knative.dev/pkg/apis" -) - -const ( - Group = "karpenter.sh" - ExtensionsGroup = "extensions." + Group - CompatabilityGroup = "compatibility." + Group - TestingGroup = "testing." + Group // Exclusively used for labeling/discovery in testing -) - -var ( - SchemeGroupVersion = schema.GroupVersion{Group: Group, Version: "v1alpha5"} - SchemeBuilder = runtime.NewSchemeBuilder(func(scheme *runtime.Scheme) error { - scheme.AddKnownTypes(SchemeGroupVersion, - &Provisioner{}, - &ProvisionerList{}, - &Machine{}, - &MachineList{}, - ) - metav1.AddToGroupVersion(scheme, SchemeGroupVersion) - return nil - }) -) - const ( - // Active is a condition implemented by all resources. It indicates that the - // controller is able to take actions: it's correctly configured, can make - // necessary API calls, and isn't disabled. - Active apis.ConditionType = "Active" + Group = "karpenter.sh" ) diff --git a/pkg/apis/v1alpha5/suite_test.go b/pkg/apis/v1alpha5/suite_test.go deleted file mode 100644 index 7bcfc9baa7..0000000000 --- a/pkg/apis/v1alpha5/suite_test.go +++ /dev/null @@ -1,727 +0,0 @@ -/* -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package v1alpha5_test - -import ( - "context" - "fmt" - "strings" - "testing" - "time" - - "k8s.io/apimachinery/pkg/api/resource" - - "github.com/Pallinder/go-randomdata" - "github.com/mitchellh/hashstructure/v2" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - "github.com/samber/lo" - . "knative.dev/pkg/logging/testing" - "knative.dev/pkg/ptr" - - . "sigs.k8s.io/karpenter/pkg/apis/v1alpha5" - "sigs.k8s.io/karpenter/pkg/test" - - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/sets" -) - -var ctx context.Context - -func TestAPIs(t *testing.T) { - ctx = TestContextWithLogger(t) - RegisterFailHandler(Fail) - RunSpecs(t, "Validation") -} - -var _ = Describe("Validation", func() { - var provisioner *Provisioner - - BeforeEach(func() { - provisioner = &Provisioner{ - ObjectMeta: metav1.ObjectMeta{Name: strings.ToLower(randomdata.SillyName())}, - Spec: ProvisionerSpec{ - ProviderRef: &MachineTemplateRef{ - Kind: "NodeTemplate", - Name: "default", - }, - }, - } - }) - - It("should fail on negative expiry ttl", func() { - provisioner.Spec.TTLSecondsUntilExpired = ptr.Int64(-1) - Expect(provisioner.Validate(ctx)).ToNot(Succeed()) - }) - It("should succeed on a missing expiry ttl", func() { - // this already is true, but to be explicit - provisioner.Spec.TTLSecondsUntilExpired = nil - Expect(provisioner.Validate(ctx)).To(Succeed()) - }) - It("should fail on negative empty ttl", func() { - provisioner.Spec.TTLSecondsAfterEmpty = ptr.Int64(-1) - Expect(provisioner.Validate(ctx)).ToNot(Succeed()) - }) - It("should succeed on a missing empty ttl", func() { - provisioner.Spec.TTLSecondsAfterEmpty = nil - Expect(provisioner.Validate(ctx)).To(Succeed()) - }) - It("should succeed on a valid empty ttl", func() { - provisioner.Spec.TTLSecondsAfterEmpty = ptr.Int64(30) - Expect(provisioner.Validate(ctx)).To(Succeed()) - }) - It("should fail if both consolidation and TTLSecondsAfterEmpty are enabled", func() { - provisioner.Spec.TTLSecondsAfterEmpty = ptr.Int64(30) - provisioner.Spec.Consolidation = &Consolidation{Enabled: ptr.Bool(true)} - Expect(provisioner.Validate(ctx)).ToNot(Succeed()) - }) - It("should succeed if consolidation is off and TTLSecondsAfterEmpty is set", func() { - provisioner.Spec.TTLSecondsAfterEmpty = ptr.Int64(30) - provisioner.Spec.Consolidation = &Consolidation{Enabled: ptr.Bool(false)} - Expect(provisioner.Validate(ctx)).To(Succeed()) - }) - It("should succeed if consolidation is on and TTLSecondsAfterEmpty is not set", func() { - provisioner.Spec.TTLSecondsAfterEmpty = nil - provisioner.Spec.Consolidation = &Consolidation{Enabled: ptr.Bool(true)} - Expect(provisioner.Validate(ctx)).To(Succeed()) - }) - - Context("Limits", func() { - It("should allow undefined limits", func() { - provisioner.Spec.Limits = &Limits{} - Expect(provisioner.Validate(ctx)).To(Succeed()) - }) - It("should allow empty limits", func() { - provisioner.Spec.Limits = &Limits{Resources: v1.ResourceList{}} - Expect(provisioner.Validate(ctx)).To(Succeed()) - }) - }) - Context("Provider", func() { - It("should not allow provider and providerRef", func() { - provisioner.Spec.Provider = &Provider{} - provisioner.Spec.ProviderRef = &MachineTemplateRef{Name: "providerRef"} - Expect(provisioner.Validate(ctx)).ToNot(Succeed()) - }) - It("should require at least one of provider and providerRef", func() { - provisioner.Spec.ProviderRef = nil - Expect(provisioner.Validate(ctx)).ToNot(Succeed()) - }) - }) - Context("Labels", func() { - It("should allow unrecognized labels", func() { - provisioner.Spec.Labels = map[string]string{"foo": randomdata.SillyName()} - Expect(provisioner.Validate(ctx)).To(Succeed()) - }) - It("should fail for the provisioner name label", func() { - provisioner.Spec.Labels = map[string]string{ProvisionerNameLabelKey: randomdata.SillyName()} - Expect(provisioner.Validate(ctx)).ToNot(Succeed()) - }) - It("should fail for invalid label keys", func() { - provisioner.Spec.Labels = map[string]string{"spaces are not allowed": randomdata.SillyName()} - Expect(provisioner.Validate(ctx)).ToNot(Succeed()) - }) - It("should fail for invalid label values", func() { - provisioner.Spec.Labels = map[string]string{randomdata.SillyName(): "/ is not allowed"} - Expect(provisioner.Validate(ctx)).ToNot(Succeed()) - }) - It("should fail for restricted label domains", func() { - for label := range RestrictedLabelDomains { - provisioner.Spec.Labels = map[string]string{label + "/unknown": randomdata.SillyName()} - Expect(provisioner.Validate(ctx)).ToNot(Succeed()) - } - }) - It("should allow labels kOps require", func() { - provisioner.Spec.Labels = map[string]string{ - "kops.k8s.io/instancegroup": "karpenter-nodes", - "kops.k8s.io/gpu": "1", - } - Expect(provisioner.Validate(ctx)).To(Succeed()) - }) - It("should allow labels in restricted domains exceptions list", func() { - for label := range LabelDomainExceptions { - provisioner.Spec.Labels = map[string]string{ - label: "test-value", - } - Expect(provisioner.Validate(ctx)).To(Succeed()) - } - }) - It("should allow labels prefixed with the restricted domain exceptions", func() { - for label := range LabelDomainExceptions { - provisioner.Spec.Labels = map[string]string{ - fmt.Sprintf("%s/key", label): "test-value", - } - Expect(provisioner.Validate(ctx)).To(Succeed()) - } - }) - }) - Context("Taints", func() { - It("should succeed for valid taints", func() { - provisioner.Spec.Taints = []v1.Taint{ - {Key: "a", Value: "b", Effect: v1.TaintEffectNoSchedule}, - {Key: "c", Value: "d", Effect: v1.TaintEffectNoExecute}, - {Key: "e", Value: "f", Effect: v1.TaintEffectPreferNoSchedule}, - {Key: "key-only", Effect: v1.TaintEffectNoExecute}, - } - Expect(provisioner.Validate(ctx)).To(Succeed()) - }) - It("should fail for invalid taint keys", func() { - provisioner.Spec.Taints = []v1.Taint{{Key: "???"}} - Expect(provisioner.Validate(ctx)).ToNot(Succeed()) - }) - It("should fail for missing taint key", func() { - provisioner.Spec.Taints = []v1.Taint{{Effect: v1.TaintEffectNoSchedule}} - Expect(provisioner.Validate(ctx)).ToNot(Succeed()) - }) - It("should fail for invalid taint value", func() { - provisioner.Spec.Taints = []v1.Taint{{Key: "invalid-value", Effect: v1.TaintEffectNoSchedule, Value: "???"}} - Expect(provisioner.Validate(ctx)).ToNot(Succeed()) - }) - It("should fail for invalid taint effect", func() { - provisioner.Spec.Taints = []v1.Taint{{Key: "invalid-effect", Effect: "???"}} - Expect(provisioner.Validate(ctx)).ToNot(Succeed()) - }) - It("should not fail for same key with different effects", func() { - provisioner.Spec.Taints = []v1.Taint{ - {Key: "a", Effect: v1.TaintEffectNoSchedule}, - {Key: "a", Effect: v1.TaintEffectNoExecute}, - } - Expect(provisioner.Validate(ctx)).To(Succeed()) - }) - It("should fail for duplicate taint key/effect pairs", func() { - provisioner.Spec.Taints = []v1.Taint{ - {Key: "a", Effect: v1.TaintEffectNoSchedule}, - {Key: "a", Effect: v1.TaintEffectNoSchedule}, - } - Expect(provisioner.Validate(ctx)).ToNot(Succeed()) - provisioner.Spec.Taints = []v1.Taint{ - {Key: "a", Effect: v1.TaintEffectNoSchedule}, - } - provisioner.Spec.StartupTaints = []v1.Taint{ - {Key: "a", Effect: v1.TaintEffectNoSchedule}, - } - Expect(provisioner.Validate(ctx)).ToNot(Succeed()) - }) - }) - Context("Requirements", func() { - It("should fail for the provisioner name label", func() { - provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ - {Key: ProvisionerNameLabelKey, Operator: v1.NodeSelectorOpIn, Values: []string{randomdata.SillyName()}}, - } - Expect(provisioner.Validate(ctx)).ToNot(Succeed()) - }) - It("should allow supported ops", func() { - provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test"}}, - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpGt, Values: []string{"1"}}, - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpLt, Values: []string{"1"}}, - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpNotIn}, - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpExists}, - } - Expect(provisioner.Validate(ctx)).To(Succeed()) - }) - It("should fail for unsupported ops", func() { - for _, op := range []v1.NodeSelectorOperator{"unknown"} { - provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: op, Values: []string{"test"}}, - } - Expect(provisioner.Validate(ctx)).ToNot(Succeed()) - } - }) - It("should fail for restricted domains", func() { - for label := range RestrictedLabelDomains { - provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ - {Key: label + "/test", Operator: v1.NodeSelectorOpIn, Values: []string{"test"}}, - } - Expect(provisioner.Validate(ctx)).ToNot(Succeed()) - } - }) - It("should allow restricted domains exceptions", func() { - for label := range LabelDomainExceptions { - provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ - {Key: label + "/test", Operator: v1.NodeSelectorOpIn, Values: []string{"test"}}, - } - Expect(provisioner.Validate(ctx)).To(Succeed()) - } - }) - It("should allow well known label exceptions", func() { - for label := range WellKnownLabels.Difference(sets.New(ProvisionerNameLabelKey)) { - provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ - {Key: label, Operator: v1.NodeSelectorOpIn, Values: []string{"test"}}, - } - Expect(provisioner.Validate(ctx)).To(Succeed()) - } - }) - It("should allow non-empty set after removing overlapped value", func() { - provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test", "foo"}}, - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpNotIn, Values: []string{"test", "bar"}}, - } - Expect(provisioner.Validate(ctx)).To(Succeed()) - }) - It("should allow empty requirements", func() { - provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{} - Expect(provisioner.Validate(ctx)).To(Succeed()) - }) - It("should fail with invalid GT or LT values", func() { - for _, requirement := range []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpGt, Values: []string{}}, - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpGt, Values: []string{"1", "2"}}, - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpGt, Values: []string{"a"}}, - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpGt, Values: []string{"-1"}}, - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpLt, Values: []string{}}, - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpLt, Values: []string{"1", "2"}}, - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpLt, Values: []string{"a"}}, - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpLt, Values: []string{"-1"}}, - } { - provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{requirement} - Expect(provisioner.Validate(ctx)).ToNot(Succeed()) - } - }) - }) - Context("KubeletConfiguration", func() { - It("should fail on kubeReserved with invalid keys", func() { - provisioner.Spec.KubeletConfiguration = &KubeletConfiguration{ - KubeReserved: v1.ResourceList{ - v1.ResourcePods: resource.MustParse("2"), - }, - } - Expect(provisioner.Validate(ctx)).ToNot(Succeed()) - }) - It("should fail on systemReserved with invalid keys", func() { - provisioner.Spec.KubeletConfiguration = &KubeletConfiguration{ - SystemReserved: v1.ResourceList{ - v1.ResourcePods: resource.MustParse("2"), - }, - } - Expect(provisioner.Validate(ctx)).ToNot(Succeed()) - }) - Context("Eviction Signals", func() { - Context("Eviction Hard", func() { - It("should succeed on evictionHard with valid keys", func() { - provisioner.Spec.KubeletConfiguration = &KubeletConfiguration{ - EvictionHard: map[string]string{ - "memory.available": "5%", - "nodefs.available": "10%", - "nodefs.inodesFree": "15%", - "imagefs.available": "5%", - "imagefs.inodesFree": "5%", - "pid.available": "5%", - }, - } - Expect(provisioner.Validate(ctx)).To(Succeed()) - }) - It("should fail on evictionHard with invalid keys", func() { - provisioner.Spec.KubeletConfiguration = &KubeletConfiguration{ - EvictionHard: map[string]string{ - "memory": "5%", - }, - } - Expect(provisioner.Validate(ctx)).ToNot(Succeed()) - }) - It("should fail on invalid formatted percentage value in evictionHard", func() { - provisioner.Spec.KubeletConfiguration = &KubeletConfiguration{ - EvictionHard: map[string]string{ - "memory.available": "5%3", - }, - } - Expect(provisioner.Validate(ctx)).ToNot(Succeed()) - }) - It("should fail on invalid percentage value (too large) in evictionHard", func() { - provisioner.Spec.KubeletConfiguration = &KubeletConfiguration{ - EvictionHard: map[string]string{ - "memory.available": "110%", - }, - } - Expect(provisioner.Validate(ctx)).ToNot(Succeed()) - }) - It("should fail on invalid quantity value in evictionHard", func() { - provisioner.Spec.KubeletConfiguration = &KubeletConfiguration{ - EvictionHard: map[string]string{ - "memory.available": "110GB", - }, - } - Expect(provisioner.Validate(ctx)).ToNot(Succeed()) - }) - }) - }) - Context("Eviction Soft", func() { - It("should succeed on evictionSoft with valid keys", func() { - provisioner.Spec.KubeletConfiguration = &KubeletConfiguration{ - EvictionSoft: map[string]string{ - "memory.available": "5%", - "nodefs.available": "10%", - "nodefs.inodesFree": "15%", - "imagefs.available": "5%", - "imagefs.inodesFree": "5%", - "pid.available": "5%", - }, - EvictionSoftGracePeriod: map[string]metav1.Duration{ - "memory.available": {Duration: time.Minute}, - "nodefs.available": {Duration: time.Second * 90}, - "nodefs.inodesFree": {Duration: time.Minute * 5}, - "imagefs.available": {Duration: time.Hour}, - "imagefs.inodesFree": {Duration: time.Hour * 24}, - "pid.available": {Duration: time.Minute}, - }, - } - Expect(provisioner.Validate(ctx)).To(Succeed()) - }) - It("should fail on evictionSoft with invalid keys", func() { - provisioner.Spec.KubeletConfiguration = &KubeletConfiguration{ - EvictionSoft: map[string]string{ - "memory": "5%", - }, - EvictionSoftGracePeriod: map[string]metav1.Duration{ - "memory": {Duration: time.Minute}, - }, - } - Expect(provisioner.Validate(ctx)).ToNot(Succeed()) - }) - It("should fail on invalid formatted percentage value in evictionSoft", func() { - provisioner.Spec.KubeletConfiguration = &KubeletConfiguration{ - EvictionSoft: map[string]string{ - "memory.available": "5%3", - }, - EvictionSoftGracePeriod: map[string]metav1.Duration{ - "memory.available": {Duration: time.Minute}, - }, - } - Expect(provisioner.Validate(ctx)).ToNot(Succeed()) - }) - It("should fail on invalid percentage value (too large) in evictionSoft", func() { - provisioner.Spec.KubeletConfiguration = &KubeletConfiguration{ - EvictionSoft: map[string]string{ - "memory.available": "110%", - }, - EvictionSoftGracePeriod: map[string]metav1.Duration{ - "memory.available": {Duration: time.Minute}, - }, - } - Expect(provisioner.Validate(ctx)).ToNot(Succeed()) - }) - It("should fail on invalid quantity value in evictionSoft", func() { - provisioner.Spec.KubeletConfiguration = &KubeletConfiguration{ - EvictionSoft: map[string]string{ - "memory.available": "110GB", - }, - EvictionSoftGracePeriod: map[string]metav1.Duration{ - "memory.available": {Duration: time.Minute}, - }, - } - Expect(provisioner.Validate(ctx)).ToNot(Succeed()) - }) - It("should fail when eviction soft doesn't have matching grace period", func() { - provisioner.Spec.KubeletConfiguration = &KubeletConfiguration{ - EvictionSoft: map[string]string{ - "memory.available": "200Mi", - }, - } - Expect(provisioner.Validate(ctx)).ToNot(Succeed()) - }) - }) - Context("GCThresholdPercent", func() { - Context("ImageGCHighThresholdPercent", func() { - It("should succeed on a imageGCHighThresholdPercent", func() { - provisioner.Spec.KubeletConfiguration = &KubeletConfiguration{ - ImageGCHighThresholdPercent: ptr.Int32(10), - } - Expect(provisioner.Validate(ctx)).To(Succeed()) - }) - It("should fail when imageGCHighThresholdPercent is less than imageGCLowThresholdPercent", func() { - provisioner.Spec.KubeletConfiguration = &KubeletConfiguration{ - ImageGCHighThresholdPercent: ptr.Int32(50), - ImageGCLowThresholdPercent: ptr.Int32(60), - } - Expect(provisioner.Validate(ctx)).ToNot(Succeed()) - }) - }) - Context("ImageGCLowThresholdPercent", func() { - It("should succeed on a imageGCLowThresholdPercent", func() { - provisioner.Spec.KubeletConfiguration = &KubeletConfiguration{ - ImageGCLowThresholdPercent: ptr.Int32(10), - } - Expect(provisioner.Validate(ctx)).To(Succeed()) - }) - It("should fail when imageGCLowThresholdPercent is greather than imageGCHighThresheldPercent", func() { - provisioner.Spec.KubeletConfiguration = &KubeletConfiguration{ - ImageGCHighThresholdPercent: ptr.Int32(50), - ImageGCLowThresholdPercent: ptr.Int32(60), - } - Expect(provisioner.Validate(ctx)).ToNot(Succeed()) - }) - }) - }) - Context("Eviction Soft Grace Period", func() { - It("should succeed on evictionSoftGracePeriod with valid keys", func() { - provisioner.Spec.KubeletConfiguration = &KubeletConfiguration{ - EvictionSoft: map[string]string{ - "memory.available": "5%", - "nodefs.available": "10%", - "nodefs.inodesFree": "15%", - "imagefs.available": "5%", - "imagefs.inodesFree": "5%", - "pid.available": "5%", - }, - EvictionSoftGracePeriod: map[string]metav1.Duration{ - "memory.available": {Duration: time.Minute}, - "nodefs.available": {Duration: time.Second * 90}, - "nodefs.inodesFree": {Duration: time.Minute * 5}, - "imagefs.available": {Duration: time.Hour}, - "imagefs.inodesFree": {Duration: time.Hour * 24}, - "pid.available": {Duration: time.Minute}, - }, - } - Expect(provisioner.Validate(ctx)).To(Succeed()) - }) - It("should fail on evictionSoftGracePeriod with invalid keys", func() { - provisioner.Spec.KubeletConfiguration = &KubeletConfiguration{ - EvictionSoftGracePeriod: map[string]metav1.Duration{ - "memory": {Duration: time.Minute}, - }, - } - Expect(provisioner.Validate(ctx)).ToNot(Succeed()) - }) - It("should fail when eviction soft grace period doesn't have matching threshold", func() { - provisioner.Spec.KubeletConfiguration = &KubeletConfiguration{ - EvictionSoftGracePeriod: map[string]metav1.Duration{ - "memory.available": {Duration: time.Minute}, - }, - } - Expect(provisioner.Validate(ctx)).ToNot(Succeed()) - }) - }) - }) -}) - -var _ = Describe("Limits", func() { - var provisioner *Provisioner - - BeforeEach(func() { - provisioner = &Provisioner{ - ObjectMeta: metav1.ObjectMeta{Name: strings.ToLower(randomdata.SillyName())}, - Spec: ProvisionerSpec{ - Limits: &Limits{ - Resources: v1.ResourceList{ - "cpu": resource.MustParse("16"), - }, - }, - }, - } - }) - - It("should work when usage is lower than limit", func() { - provisioner.Status.Resources = v1.ResourceList{"cpu": resource.MustParse("15")} - Expect(provisioner.Spec.Limits.ExceededBy(provisioner.Status.Resources)).To(Succeed()) - }) - It("should work when usage is equal to limit", func() { - provisioner.Status.Resources = v1.ResourceList{"cpu": resource.MustParse("16")} - Expect(provisioner.Spec.Limits.ExceededBy(provisioner.Status.Resources)).To(Succeed()) - }) - It("should fail when usage is higher than limit", func() { - provisioner.Status.Resources = v1.ResourceList{"cpu": resource.MustParse("17")} - Expect(provisioner.Spec.Limits.ExceededBy(provisioner.Status.Resources)).To(MatchError("cpu resource usage of 17 exceeds limit of 16")) - }) -}) - -var _ = Describe("Provisioner Annotation", func() { - var testProvisionerOptions test.ProvisionerOptions - var provisioner *Provisioner - - const baseProvisionerExpectedHash = "775825888864018614" - - BeforeEach(func() { - taints := []v1.Taint{ - { - Key: "keyValue1", - Effect: v1.TaintEffectNoExecute, - }, - { - Key: "keyValue2", - Effect: v1.TaintEffectNoExecute, - }, - } - testProvisionerOptions = test.ProvisionerOptions{ - Taints: taints, - StartupTaints: taints, - Labels: map[string]string{ - "keyLabel": "valueLabel", - "keyLabel2": "valueLabel2", - }, - Kubelet: &KubeletConfiguration{ - MaxPods: ptr.Int32(10), - }, - Annotations: map[string]string{ - "keyAnnotation": "valueAnnotation", - "keyAnnotation2": "valueAnnotation2", - }, - } - provisioner = test.Provisioner(testProvisionerOptions) - }) - DescribeTable( - "Static Hash Values", - func(expectedHash string, opts ...test.ProvisionerOptions) { - overrides := append([]test.ProvisionerOptions{testProvisionerOptions}, opts...) - p := test.Provisioner(overrides...) - Expect(p.Hash()).To(Equal(expectedHash)) - }, - // Base provisioner - Entry("should match with the base provisioner", baseProvisionerExpectedHash), - - // Modified static fields - expect change from base provisioner - Entry( - "should match with modified annotations", - "8291486979066679783", - test.ProvisionerOptions{Annotations: map[string]string{"keyAnnotationTest": "valueAnnotationTest"}}, - ), - Entry( - "should match with modified labels", - "13084558173553732887", - test.ProvisionerOptions{Labels: map[string]string{"keyLabelTest": "valueLabelTest"}}, - ), - Entry( - "should match with modified taints", - "2448241124432930761", - test.ProvisionerOptions{Taints: []v1.Taint{{Key: "keytest2Taint", Effect: v1.TaintEffectNoExecute}}}, - ), - Entry( - "should match with modified startup taints", - "6798849987795493190", - test.ProvisionerOptions{StartupTaints: []v1.Taint{{Key: "keytest2StartupTaint", Effect: v1.TaintEffectNoExecute}}}, - ), - Entry( - "should match with modified kubelet config", - "12466819533084384505", - test.ProvisionerOptions{Kubelet: &KubeletConfiguration{MaxPods: ptr.Int32(30)}}, - ), - - // Modified behavior fields - shouldn't change from base provisioner - Entry( - "should match with modified limits", - baseProvisionerExpectedHash, - test.ProvisionerOptions{Limits: v1.ResourceList{"cpu": resource.MustParse("4")}}, - ), - Entry( - "should match with modified provider ref", - baseProvisionerExpectedHash, - test.ProvisionerOptions{ProviderRef: &MachineTemplateRef{Name: "foobar"}}, - ), - Entry( - "should match with modified requirements", - baseProvisionerExpectedHash, - test.ProvisionerOptions{Requirements: []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test"}}, - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpGt, Values: []string{"1"}}, - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpLt, Values: []string{"1"}}, - }}, - ), - Entry( - "should match with modified TTLSecondsUntilExpired", - baseProvisionerExpectedHash, - test.ProvisionerOptions{TTLSecondsUntilExpired: lo.ToPtr(int64(30))}, - ), - Entry( - "should match with modified TTLSecondsAfterEmpty", - baseProvisionerExpectedHash, - test.ProvisionerOptions{TTLSecondsAfterEmpty: lo.ToPtr(int64(50))}, - ), - Entry( - "should match with modified weight", - baseProvisionerExpectedHash, - test.ProvisionerOptions{Weight: lo.ToPtr(int32(80))}, - ), - Entry( - "should match with modified consolidation flag", - baseProvisionerExpectedHash, - test.ProvisionerOptions{Consolidation: &Consolidation{lo.ToPtr(true)}}, - ), - ) - It("should change hash when static fields are updated", func() { - expectedHash := provisioner.Hash() - - // Change one static field for 5 provisioners - provisionerFieldToChange := []*Provisioner{ - test.Provisioner(testProvisionerOptions, test.ProvisionerOptions{Annotations: map[string]string{"keyAnnotationTest": "valueAnnotationTest"}}), - test.Provisioner(testProvisionerOptions, test.ProvisionerOptions{Labels: map[string]string{"keyLabelTest": "valueLabelTest"}}), - test.Provisioner(testProvisionerOptions, test.ProvisionerOptions{Taints: []v1.Taint{{Key: "keytest2Taint", Effect: v1.TaintEffectNoExecute}}}), - test.Provisioner(testProvisionerOptions, test.ProvisionerOptions{StartupTaints: []v1.Taint{{Key: "keytest2StartupTaint", Effect: v1.TaintEffectNoExecute}}}), - test.Provisioner(testProvisionerOptions, test.ProvisionerOptions{Kubelet: &KubeletConfiguration{MaxPods: ptr.Int32(30)}}), - } - - for _, updatedProvisioner := range provisionerFieldToChange { - actualHash := updatedProvisioner.Hash() - Expect(actualHash).ToNot(Equal(fmt.Sprint(expectedHash))) - } - }) - It("should not change hash when behavior fields are updated", func() { - actualHash := provisioner.Hash() - - expectedHash, err := hashstructure.Hash(provisioner.Spec, hashstructure.FormatV2, &hashstructure.HashOptions{ - SlicesAsSets: true, - IgnoreZeroValue: true, - ZeroNil: true, - }) - Expect(err).ToNot(HaveOccurred()) - Expect(actualHash).To(Equal(fmt.Sprint(expectedHash))) - - // Update a behavior field - provisioner.Spec.Limits = &Limits{Resources: v1.ResourceList{"cpu": resource.MustParse("16")}} - provisioner.Spec.Consolidation = &Consolidation{Enabled: lo.ToPtr(true)} - provisioner.Spec.TTLSecondsAfterEmpty = lo.ToPtr(int64(30)) - provisioner.Spec.TTLSecondsUntilExpired = lo.ToPtr(int64(50)) - provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test"}}, - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpGt, Values: []string{"1"}}, - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpLt, Values: []string{"1"}}, - } - provisioner.Spec.Weight = lo.ToPtr(int32(80)) - - actualHash = provisioner.Hash() - Expect(err).ToNot(HaveOccurred()) - Expect(actualHash).To(Equal(fmt.Sprint(expectedHash))) - }) - It("should expect two provisioner with the same spec to have the same provisioner hash", func() { - provisionerTwo := &Provisioner{ - ObjectMeta: metav1.ObjectMeta{Name: strings.ToLower(randomdata.SillyName())}, - } - provisionerTwo.Spec = provisioner.Spec - - Expect(provisioner.Hash()).To(Equal(provisionerTwo.Hash())) - }) - It("should expect hashes that are reordered to not produce a new hash", func() { - expectedHash := provisioner.Hash() - updatedTaints := []v1.Taint{ - { - Key: "keyValue2", - Effect: v1.TaintEffectNoExecute, - }, - { - Key: "keyValue1", - Effect: v1.TaintEffectNoExecute, - }, - } - - provisionerFieldToChange := []*Provisioner{ - test.Provisioner(testProvisionerOptions, test.ProvisionerOptions{Annotations: map[string]string{"keyAnnotation2": "valueAnnotation2", "keyAnnotation": "valueAnnotation"}}), - test.Provisioner(testProvisionerOptions, test.ProvisionerOptions{Taints: updatedTaints}), - test.Provisioner(testProvisionerOptions, test.ProvisionerOptions{StartupTaints: updatedTaints}), - } - - for _, updatedProvisioner := range provisionerFieldToChange { - actualHash := updatedProvisioner.Hash() - Expect(actualHash).To(Equal(fmt.Sprint(expectedHash))) - } - }) -}) diff --git a/pkg/apis/v1alpha5/zz_generated.deepcopy.go b/pkg/apis/v1alpha5/zz_generated.deepcopy.go deleted file mode 100644 index 1b1a993118..0000000000 --- a/pkg/apis/v1alpha5/zz_generated.deepcopy.go +++ /dev/null @@ -1,520 +0,0 @@ -//go:build !ignore_autogenerated - -/* -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Code generated by controller-gen. DO NOT EDIT. - -package v1alpha5 - -import ( - "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "knative.dev/pkg/apis" -) - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *Consolidation) DeepCopyInto(out *Consolidation) { - *out = *in - if in.Enabled != nil { - in, out := &in.Enabled, &out.Enabled - *out = new(bool) - **out = **in - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Consolidation. -func (in *Consolidation) DeepCopy() *Consolidation { - if in == nil { - return nil - } - out := new(Consolidation) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *KubeletConfiguration) DeepCopyInto(out *KubeletConfiguration) { - *out = *in - if in.ClusterDNS != nil { - in, out := &in.ClusterDNS, &out.ClusterDNS - *out = make([]string, len(*in)) - copy(*out, *in) - } - if in.ContainerRuntime != nil { - in, out := &in.ContainerRuntime, &out.ContainerRuntime - *out = new(string) - **out = **in - } - if in.MaxPods != nil { - in, out := &in.MaxPods, &out.MaxPods - *out = new(int32) - **out = **in - } - if in.PodsPerCore != nil { - in, out := &in.PodsPerCore, &out.PodsPerCore - *out = new(int32) - **out = **in - } - if in.SystemReserved != nil { - in, out := &in.SystemReserved, &out.SystemReserved - *out = make(v1.ResourceList, len(*in)) - for key, val := range *in { - (*out)[key] = val.DeepCopy() - } - } - if in.KubeReserved != nil { - in, out := &in.KubeReserved, &out.KubeReserved - *out = make(v1.ResourceList, len(*in)) - for key, val := range *in { - (*out)[key] = val.DeepCopy() - } - } - if in.EvictionHard != nil { - in, out := &in.EvictionHard, &out.EvictionHard - *out = make(map[string]string, len(*in)) - for key, val := range *in { - (*out)[key] = val - } - } - if in.EvictionSoft != nil { - in, out := &in.EvictionSoft, &out.EvictionSoft - *out = make(map[string]string, len(*in)) - for key, val := range *in { - (*out)[key] = val - } - } - if in.EvictionSoftGracePeriod != nil { - in, out := &in.EvictionSoftGracePeriod, &out.EvictionSoftGracePeriod - *out = make(map[string]metav1.Duration, len(*in)) - for key, val := range *in { - (*out)[key] = val - } - } - if in.EvictionMaxPodGracePeriod != nil { - in, out := &in.EvictionMaxPodGracePeriod, &out.EvictionMaxPodGracePeriod - *out = new(int32) - **out = **in - } - if in.ImageGCHighThresholdPercent != nil { - in, out := &in.ImageGCHighThresholdPercent, &out.ImageGCHighThresholdPercent - *out = new(int32) - **out = **in - } - if in.ImageGCLowThresholdPercent != nil { - in, out := &in.ImageGCLowThresholdPercent, &out.ImageGCLowThresholdPercent - *out = new(int32) - **out = **in - } - if in.CPUCFSQuota != nil { - in, out := &in.CPUCFSQuota, &out.CPUCFSQuota - *out = new(bool) - **out = **in - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubeletConfiguration. -func (in *KubeletConfiguration) DeepCopy() *KubeletConfiguration { - if in == nil { - return nil - } - out := new(KubeletConfiguration) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *Limits) DeepCopyInto(out *Limits) { - *out = *in - if in.Resources != nil { - in, out := &in.Resources, &out.Resources - *out = make(v1.ResourceList, len(*in)) - for key, val := range *in { - (*out)[key] = val.DeepCopy() - } - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Limits. -func (in *Limits) DeepCopy() *Limits { - if in == nil { - return nil - } - out := new(Limits) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *Machine) DeepCopyInto(out *Machine) { - *out = *in - out.TypeMeta = in.TypeMeta - in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - in.Spec.DeepCopyInto(&out.Spec) - in.Status.DeepCopyInto(&out.Status) -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Machine. -func (in *Machine) DeepCopy() *Machine { - if in == nil { - return nil - } - out := new(Machine) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *Machine) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *MachineList) DeepCopyInto(out *MachineList) { - *out = *in - out.TypeMeta = in.TypeMeta - in.ListMeta.DeepCopyInto(&out.ListMeta) - if in.Items != nil { - in, out := &in.Items, &out.Items - *out = make([]Machine, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MachineList. -func (in *MachineList) DeepCopy() *MachineList { - if in == nil { - return nil - } - out := new(MachineList) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *MachineList) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *MachineSpec) DeepCopyInto(out *MachineSpec) { - *out = *in - if in.Taints != nil { - in, out := &in.Taints, &out.Taints - *out = make([]v1.Taint, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } - if in.StartupTaints != nil { - in, out := &in.StartupTaints, &out.StartupTaints - *out = make([]v1.Taint, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } - if in.Requirements != nil { - in, out := &in.Requirements, &out.Requirements - *out = make([]v1.NodeSelectorRequirement, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } - in.Resources.DeepCopyInto(&out.Resources) - if in.Kubelet != nil { - in, out := &in.Kubelet, &out.Kubelet - *out = new(KubeletConfiguration) - (*in).DeepCopyInto(*out) - } - if in.MachineTemplateRef != nil { - in, out := &in.MachineTemplateRef, &out.MachineTemplateRef - *out = new(MachineTemplateRef) - **out = **in - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MachineSpec. -func (in *MachineSpec) DeepCopy() *MachineSpec { - if in == nil { - return nil - } - out := new(MachineSpec) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *MachineStatus) DeepCopyInto(out *MachineStatus) { - *out = *in - if in.Capacity != nil { - in, out := &in.Capacity, &out.Capacity - *out = make(v1.ResourceList, len(*in)) - for key, val := range *in { - (*out)[key] = val.DeepCopy() - } - } - if in.Allocatable != nil { - in, out := &in.Allocatable, &out.Allocatable - *out = make(v1.ResourceList, len(*in)) - for key, val := range *in { - (*out)[key] = val.DeepCopy() - } - } - if in.Conditions != nil { - in, out := &in.Conditions, &out.Conditions - *out = make(apis.Conditions, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MachineStatus. -func (in *MachineStatus) DeepCopy() *MachineStatus { - if in == nil { - return nil - } - out := new(MachineStatus) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *MachineTemplateRef) DeepCopyInto(out *MachineTemplateRef) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MachineTemplateRef. -func (in *MachineTemplateRef) DeepCopy() *MachineTemplateRef { - if in == nil { - return nil - } - out := new(MachineTemplateRef) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *Provisioner) DeepCopyInto(out *Provisioner) { - *out = *in - out.TypeMeta = in.TypeMeta - in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - in.Spec.DeepCopyInto(&out.Spec) - in.Status.DeepCopyInto(&out.Status) -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Provisioner. -func (in *Provisioner) DeepCopy() *Provisioner { - if in == nil { - return nil - } - out := new(Provisioner) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *Provisioner) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ProvisionerList) DeepCopyInto(out *ProvisionerList) { - *out = *in - out.TypeMeta = in.TypeMeta - in.ListMeta.DeepCopyInto(&out.ListMeta) - if in.Items != nil { - in, out := &in.Items, &out.Items - *out = make([]Provisioner, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProvisionerList. -func (in *ProvisionerList) DeepCopy() *ProvisionerList { - if in == nil { - return nil - } - out := new(ProvisionerList) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *ProvisionerList) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ProvisionerSpec) DeepCopyInto(out *ProvisionerSpec) { - *out = *in - if in.Annotations != nil { - in, out := &in.Annotations, &out.Annotations - *out = make(map[string]string, len(*in)) - for key, val := range *in { - (*out)[key] = val - } - } - if in.Labels != nil { - in, out := &in.Labels, &out.Labels - *out = make(map[string]string, len(*in)) - for key, val := range *in { - (*out)[key] = val - } - } - if in.Taints != nil { - in, out := &in.Taints, &out.Taints - *out = make([]v1.Taint, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } - if in.StartupTaints != nil { - in, out := &in.StartupTaints, &out.StartupTaints - *out = make([]v1.Taint, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } - if in.Requirements != nil { - in, out := &in.Requirements, &out.Requirements - *out = make([]v1.NodeSelectorRequirement, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } - if in.KubeletConfiguration != nil { - in, out := &in.KubeletConfiguration, &out.KubeletConfiguration - *out = new(KubeletConfiguration) - (*in).DeepCopyInto(*out) - } - if in.Provider != nil { - in, out := &in.Provider, &out.Provider - *out = new(runtime.RawExtension) - (*in).DeepCopyInto(*out) - } - if in.ProviderRef != nil { - in, out := &in.ProviderRef, &out.ProviderRef - *out = new(MachineTemplateRef) - **out = **in - } - if in.TTLSecondsAfterEmpty != nil { - in, out := &in.TTLSecondsAfterEmpty, &out.TTLSecondsAfterEmpty - *out = new(int64) - **out = **in - } - if in.TTLSecondsUntilExpired != nil { - in, out := &in.TTLSecondsUntilExpired, &out.TTLSecondsUntilExpired - *out = new(int64) - **out = **in - } - if in.Limits != nil { - in, out := &in.Limits, &out.Limits - *out = new(Limits) - (*in).DeepCopyInto(*out) - } - if in.Weight != nil { - in, out := &in.Weight, &out.Weight - *out = new(int32) - **out = **in - } - if in.Consolidation != nil { - in, out := &in.Consolidation, &out.Consolidation - *out = new(Consolidation) - (*in).DeepCopyInto(*out) - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProvisionerSpec. -func (in *ProvisionerSpec) DeepCopy() *ProvisionerSpec { - if in == nil { - return nil - } - out := new(ProvisionerSpec) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ProvisionerStatus) DeepCopyInto(out *ProvisionerStatus) { - *out = *in - if in.LastScaleTime != nil { - in, out := &in.LastScaleTime, &out.LastScaleTime - *out = new(apis.VolatileTime) - (*in).DeepCopyInto(*out) - } - if in.Conditions != nil { - in, out := &in.Conditions, &out.Conditions - *out = make(apis.Conditions, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } - if in.Resources != nil { - in, out := &in.Resources, &out.Resources - *out = make(v1.ResourceList, len(*in)) - for key, val := range *in { - (*out)[key] = val.DeepCopy() - } - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProvisionerStatus. -func (in *ProvisionerStatus) DeepCopy() *ProvisionerStatus { - if in == nil { - return nil - } - out := new(ProvisionerStatus) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ResourceRequirements) DeepCopyInto(out *ResourceRequirements) { - *out = *in - if in.Requests != nil { - in, out := &in.Requests, &out.Requests - *out = make(v1.ResourceList, len(*in)) - for key, val := range *in { - (*out)[key] = val.DeepCopy() - } - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ResourceRequirements. -func (in *ResourceRequirements) DeepCopy() *ResourceRequirements { - if in == nil { - return nil - } - out := new(ResourceRequirements) - in.DeepCopyInto(out) - return out -} diff --git a/pkg/apis/v1beta1/labels.go b/pkg/apis/v1beta1/labels.go index 49e91ffb0f..745993e7d1 100644 --- a/pkg/apis/v1beta1/labels.go +++ b/pkg/apis/v1beta1/labels.go @@ -69,7 +69,7 @@ var ( // WellKnownLabels are labels that belong to the RestrictedLabelDomains but allowed. // Karpenter is aware of these labels, and they can be used to further narrow down - // the range of the corresponding values by either provisioner or pods. + // the range of the corresponding values by either nodepool or pods. WellKnownLabels = sets.New( NodePoolLabelKey, v1.LabelTopologyZone, diff --git a/pkg/apis/v1beta1/nodeclaim_validation.go b/pkg/apis/v1beta1/nodeclaim_validation.go index 4e3949e116..8235a3e15f 100644 --- a/pkg/apis/v1beta1/nodeclaim_validation.go +++ b/pkg/apis/v1beta1/nodeclaim_validation.go @@ -127,8 +127,8 @@ func (in *NodeClaimSpec) validateTaintsField(taints []v1.Taint, existing map[tai return errs } -// This function is used by the NodeClaim validation webhook to verify the provisioner requirements. -// When this function is called, the provisioner's requirements do not include the requirements from labels. +// This function is used by the NodeClaim validation webhook to verify the nodepool requirements. +// When this function is called, the nodepool's requirements do not include the requirements from labels. // NodeClaim requirements only support well known labels. func (in *NodeClaimSpec) validateRequirements() (errs *apis.FieldError) { for i, requirement := range in.Requirements { diff --git a/pkg/apis/v1beta1/nodepool_validation_webhook_test.go b/pkg/apis/v1beta1/nodepool_validation_webhook_test.go index 2dfc6bbb2e..ff97659032 100644 --- a/pkg/apis/v1beta1/nodepool_validation_webhook_test.go +++ b/pkg/apis/v1beta1/nodepool_validation_webhook_test.go @@ -515,10 +515,10 @@ var _ = Describe("Webhook/Validation", func() { }) var _ = Describe("Limits", func() { - var provisioner *NodePool + var nodepool *NodePool BeforeEach(func() { - provisioner = &NodePool{ + nodepool = &NodePool{ ObjectMeta: metav1.ObjectMeta{Name: strings.ToLower(randomdata.SillyName())}, Spec: NodePoolSpec{ Limits: Limits(v1.ResourceList{ @@ -529,15 +529,15 @@ var _ = Describe("Limits", func() { }) It("should work when usage is lower than limit", func() { - provisioner.Status.Resources = v1.ResourceList{"cpu": resource.MustParse("15")} - Expect(provisioner.Spec.Limits.ExceededBy(provisioner.Status.Resources)).To(Succeed()) + nodepool.Status.Resources = v1.ResourceList{"cpu": resource.MustParse("15")} + Expect(nodepool.Spec.Limits.ExceededBy(nodepool.Status.Resources)).To(Succeed()) }) It("should work when usage is equal to limit", func() { - provisioner.Status.Resources = v1.ResourceList{"cpu": resource.MustParse("16")} - Expect(provisioner.Spec.Limits.ExceededBy(provisioner.Status.Resources)).To(Succeed()) + nodepool.Status.Resources = v1.ResourceList{"cpu": resource.MustParse("16")} + Expect(nodepool.Spec.Limits.ExceededBy(nodepool.Status.Resources)).To(Succeed()) }) It("should fail when usage is higher than limit", func() { - provisioner.Status.Resources = v1.ResourceList{"cpu": resource.MustParse("17")} - Expect(provisioner.Spec.Limits.ExceededBy(provisioner.Status.Resources)).To(MatchError("cpu resource usage of 17 exceeds limit of 16")) + nodepool.Status.Resources = v1.ResourceList{"cpu": resource.MustParse("17")} + Expect(nodepool.Spec.Limits.ExceededBy(nodepool.Status.Resources)).To(MatchError("cpu resource usage of 17 exceeds limit of 16")) }) }) diff --git a/pkg/cloudprovider/fake/instancetype.go b/pkg/cloudprovider/fake/instancetype.go index 52ba8ea63d..a19b72ce7b 100644 --- a/pkg/cloudprovider/fake/instancetype.go +++ b/pkg/cloudprovider/fake/instancetype.go @@ -24,7 +24,6 @@ import ( v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" - "sigs.k8s.io/karpenter/pkg/apis/v1alpha5" "sigs.k8s.io/karpenter/pkg/apis/v1beta1" "sigs.k8s.io/karpenter/pkg/cloudprovider" "sigs.k8s.io/karpenter/pkg/scheduling" @@ -79,7 +78,7 @@ func NewInstanceType(options InstanceTypeOptions) *cloudprovider.InstanceType { scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, options.Architecture), scheduling.NewRequirement(v1.LabelOSStable, v1.NodeSelectorOpIn, sets.List(options.OperatingSystems)...), scheduling.NewRequirement(v1.LabelTopologyZone, v1.NodeSelectorOpIn, lo.Map(options.Offerings.Available(), func(o cloudprovider.Offering, _ int) string { return o.Zone })...), - scheduling.NewRequirement(v1alpha5.LabelCapacityType, v1.NodeSelectorOpIn, lo.Map(options.Offerings.Available(), func(o cloudprovider.Offering, _ int) string { return o.CapacityType })...), + scheduling.NewRequirement(v1beta1.CapacityTypeLabelKey, v1.NodeSelectorOpIn, lo.Map(options.Offerings.Available(), func(o cloudprovider.Offering, _ int) string { return o.CapacityType })...), scheduling.NewRequirement(LabelInstanceSize, v1.NodeSelectorOpDoesNotExist), scheduling.NewRequirement(ExoticInstanceLabelKey, v1.NodeSelectorOpDoesNotExist), scheduling.NewRequirement(IntegerInstanceLabelKey, v1.NodeSelectorOpIn, fmt.Sprint(options.Resources.Cpu().Value())), @@ -112,9 +111,9 @@ func InstanceTypesAssorted() []*cloudprovider.InstanceType { for _, cpu := range []int{1, 2, 4, 8, 16, 32, 64} { for _, mem := range []int{1, 2, 4, 8, 16, 32, 64, 128} { for _, zone := range []string{"test-zone-1", "test-zone-2", "test-zone-3"} { - for _, ct := range []string{v1alpha5.CapacityTypeSpot, v1alpha5.CapacityTypeOnDemand} { + for _, ct := range []string{v1beta1.CapacityTypeSpot, v1beta1.CapacityTypeOnDemand} { for _, os := range []sets.Set[string]{sets.New(string(v1.Linux)), sets.New(string(v1.Windows))} { - for _, arch := range []string{v1alpha5.ArchitectureAmd64, v1alpha5.ArchitectureArm64} { + for _, arch := range []string{v1beta1.ArchitectureAmd64, v1beta1.ArchitectureArm64} { opts := InstanceTypeOptions{ Name: fmt.Sprintf("%d-cpu-%d-mem-%s-%s-%s-%s", cpu, mem, arch, strings.Join(sets.List(os), ","), zone, ct), Architecture: arch, diff --git a/pkg/cloudprovider/metrics/cloudprovider_test.go b/pkg/cloudprovider/metrics/cloudprovider_test.go index 2e18cc90ff..b81277f061 100644 --- a/pkg/cloudprovider/metrics/cloudprovider_test.go +++ b/pkg/cloudprovider/metrics/cloudprovider_test.go @@ -30,7 +30,7 @@ var _ = Describe("Cloudprovider", func() { var nodeClassNotReadyErr = cloudprovider.NewNodeClassNotReadyError(errors.New("not ready")) var unknownErr = errors.New("this is an error we don't know about") - Describe("CloudProvider machine errors via GetErrorTypeLabelValue()", func() { + Describe("CloudProvider nodeclaim errors via GetErrorTypeLabelValue()", func() { Context("when the error is known", func() { It("nodeclaim not found should be recognized", func() { Expect(metrics.GetErrorTypeLabelValue(nodeClaimNotFoundErr)).To(Equal(metrics.NodeClaimNotFoundError)) diff --git a/pkg/cloudprovider/types.go b/pkg/cloudprovider/types.go index 0cb2066fd7..a488354ead 100644 --- a/pkg/cloudprovider/types.go +++ b/pkg/cloudprovider/types.go @@ -44,7 +44,7 @@ type CloudProvider interface { // List retrieves all NodeClaims from the cloudprovider List(context.Context) ([]*v1beta1.NodeClaim, error) // GetInstanceTypes returns instance types supported by the cloudprovider. - // Availability of types or zone may vary by provisioner or over time. Regardless of + // Availability of types or zone may vary by nodepool or over time. Regardless of // availability, the GetInstanceTypes method should always return all instance types, // even those with no offerings available. GetInstanceTypes(context.Context, *v1beta1.NodePool) ([]*InstanceType, error) diff --git a/pkg/controllers/disruption/consolidation_test.go b/pkg/controllers/disruption/consolidation_test.go index d74b13a6b8..503dfd2f29 100644 --- a/pkg/controllers/disruption/consolidation_test.go +++ b/pkg/controllers/disruption/consolidation_test.go @@ -838,7 +838,7 @@ var _ = Describe("Consolidation", func() { ExpectReconcileSucceeded(ctx, queue, types.NamespacedName{}) - // Cascade any deletion of the machine to the node + // Cascade any deletion of the nodeclaim to the node ExpectNodeClaimsCascadeDeletion(ctx, env.Client, nodeClaim) // we should delete the non-annotated node and replace with a cheaper node @@ -851,7 +851,7 @@ var _ = Describe("Consolidation", func() { Name: "current-on-demand", Offerings: []cloudprovider.Offering{ { - CapacityType: v1alpha5.CapacityTypeOnDemand, + CapacityType: v1beta1.CapacityTypeOnDemand, Zone: "test-zone-1a", Price: 0.5, Available: false, @@ -862,19 +862,19 @@ var _ = Describe("Consolidation", func() { Name: "potential-spot-replacement", Offerings: []cloudprovider.Offering{ { - CapacityType: v1alpha5.CapacityTypeSpot, + CapacityType: v1beta1.CapacityTypeSpot, Zone: "test-zone-1a", Price: 1.0, Available: true, }, { - CapacityType: v1alpha5.CapacityTypeSpot, + CapacityType: v1beta1.CapacityTypeSpot, Zone: "test-zone-1b", Price: 0.2, Available: true, }, { - CapacityType: v1alpha5.CapacityTypeSpot, + CapacityType: v1beta1.CapacityTypeSpot, Zone: "test-zone-1c", Price: 0.4, Available: true, @@ -946,7 +946,7 @@ var _ = Describe("Consolidation", func() { Name: "current-on-demand", Offerings: []cloudprovider.Offering{ { - CapacityType: v1alpha5.CapacityTypeOnDemand, + CapacityType: v1beta1.CapacityTypeOnDemand, Zone: "test-zone-1a", Price: 0.5, Available: false, @@ -957,25 +957,25 @@ var _ = Describe("Consolidation", func() { Name: "on-demand-replacement", Offerings: []cloudprovider.Offering{ { - CapacityType: v1alpha5.CapacityTypeOnDemand, + CapacityType: v1beta1.CapacityTypeOnDemand, Zone: "test-zone-1a", Price: 0.6, Available: true, }, { - CapacityType: v1alpha5.CapacityTypeOnDemand, + CapacityType: v1beta1.CapacityTypeOnDemand, Zone: "test-zone-1b", Price: 0.6, Available: true, }, { - CapacityType: v1alpha5.CapacityTypeSpot, + CapacityType: v1beta1.CapacityTypeSpot, Zone: "test-zone-1b", Price: 0.2, Available: true, }, { - CapacityType: v1alpha5.CapacityTypeSpot, + CapacityType: v1beta1.CapacityTypeSpot, Zone: "test-zone-1c", Price: 0.3, Available: true, @@ -1014,7 +1014,7 @@ var _ = Describe("Consolidation", func() { { Key: v1beta1.CapacityTypeLabelKey, Operator: v1.NodeSelectorOpIn, - Values: []string{v1alpha5.CapacityTypeOnDemand}, + Values: []string{v1beta1.CapacityTypeOnDemand}, }, } nodeClaim, node = test.NodeClaimAndNode(v1beta1.NodeClaim{ @@ -1160,8 +1160,8 @@ var _ = Describe("Consolidation", func() { BlockOwnerDeletion: ptr.Bool(true), }, }}}) - nodeTemplateProvisioner := test.Provisioner() - nodeTemplateProvisioner.Spec.Provider = nil + nodeClassNodePool := test.NodePool() + nodeClassNodePool.Spec.Template.Spec.NodeClassRef = nil ExpectApplied(ctx, env.Client, rs, pods[0], pods[1], pods[2], nodeClaim, node, nodeClaim2, node2, nodePool) // bind pods to node @@ -1517,7 +1517,7 @@ var _ = Describe("Consolidation", func() { ExpectReconcileSucceeded(ctx, queue, types.NamespacedName{}) - // Cascade any deletion of the machine to the node + // Cascade any deletion of the nodeclaim to the node ExpectNodeClaimsCascadeDeletion(ctx, env.Client, nodeClaim) // we should delete the non-annotated node @@ -1941,7 +1941,7 @@ var _ = Describe("Consolidation", func() { // Process the item so that the nodes can be deleted. ExpectReconcileSucceeded(ctx, queue, types.NamespacedName{}) - // Cascade any deletion of the machine to the node + // Cascade any deletion of the nodeclaim to the node ExpectNodeClaimsCascadeDeletion(ctx, env.Client, nodeClaim2) // we don't need a new node, but we should evict everything off one of node2 which only has a single pod diff --git a/pkg/controllers/metrics/node/controller.go b/pkg/controllers/metrics/node/controller.go index 4b5ab942a3..dabccb9a7b 100644 --- a/pkg/controllers/metrics/node/controller.go +++ b/pkg/controllers/metrics/node/controller.go @@ -28,7 +28,6 @@ import ( crmetrics "sigs.k8s.io/controller-runtime/pkg/metrics" "sigs.k8s.io/controller-runtime/pkg/reconcile" - "sigs.k8s.io/karpenter/pkg/apis/v1alpha5" "sigs.k8s.io/karpenter/pkg/apis/v1beta1" "sigs.k8s.io/karpenter/pkg/controllers/state" "sigs.k8s.io/karpenter/pkg/metrics" @@ -37,10 +36,9 @@ import ( ) const ( - resourceType = "resource_type" - nodeName = "node_name" - nodeProvisioner = "provisioner" - nodePhase = "phase" + resourceType = "resource_type" + nodeName = "node_name" + nodePhase = "phase" ) var ( @@ -106,7 +104,6 @@ func nodeLabelNames() []string { sets.New(lo.Values(wellKnownLabels)...).UnsortedList(), resourceType, nodeName, - nodeProvisioner, nodePhase, ) } @@ -176,7 +173,6 @@ func getNodeLabels(node *v1.Node, resourceTypeName string) prometheus.Labels { metricLabels := prometheus.Labels{} metricLabels[resourceType] = resourceTypeName metricLabels[nodeName] = node.Name - metricLabels[nodeProvisioner] = node.Labels[v1alpha5.ProvisionerNameLabelKey] metricLabels[nodePhase] = string(node.Status.Phase) // Populate well known labels @@ -189,7 +185,7 @@ func getNodeLabels(node *v1.Node, resourceTypeName string) prometheus.Labels { func getWellKnownLabels() map[string]string { labels := make(map[string]string) // TODO @joinnis: Remove v1alpha5 well-known labels in favor of only v1beta1 well-known labels after v1alpha5 is dropped - for wellKnownLabel := range v1alpha5.WellKnownLabels.Union(v1beta1.WellKnownLabels) { + for wellKnownLabel := range v1beta1.WellKnownLabels { if parts := strings.Split(wellKnownLabel, "/"); len(parts) == 2 { label := parts[1] // Reformat label names to be consistent with Prometheus naming conventions (snake_case) diff --git a/pkg/controllers/metrics/nodepool/controller.go b/pkg/controllers/metrics/nodepool/controller.go index bfa35d0832..30014fe9af 100644 --- a/pkg/controllers/metrics/nodepool/controller.go +++ b/pkg/controllers/metrics/nodepool/controller.go @@ -48,7 +48,7 @@ var ( Namespace: metrics.Namespace, Subsystem: nodePoolSubsystem, Name: "limit", - Help: "The nodepool limits are the limits specified on the provisioner that restrict the quantity of resources provisioned. Labeled by nodepool name and resource type.", + Help: "The nodepool limits are the limits specified on the nodepool that restrict the quantity of resources provisioned. Labeled by nodepool name and resource type.", }, []string{ resourceTypeLabel, diff --git a/pkg/controllers/metrics/pod/controller.go b/pkg/controllers/metrics/pod/controller.go index 5e757b88cd..1f0af670a6 100644 --- a/pkg/controllers/metrics/pod/controller.go +++ b/pkg/controllers/metrics/pod/controller.go @@ -32,7 +32,6 @@ import ( crmetrics "sigs.k8s.io/controller-runtime/pkg/metrics" "sigs.k8s.io/controller-runtime/pkg/reconcile" - "sigs.k8s.io/karpenter/pkg/apis/v1alpha5" "sigs.k8s.io/karpenter/pkg/apis/v1beta1" "sigs.k8s.io/karpenter/pkg/metrics" "sigs.k8s.io/karpenter/pkg/operator/controller" @@ -43,7 +42,6 @@ const ( podNameSpace = "namespace" ownerSelfLink = "owner" podHostName = "node" - podProvisioner = "provisioner" podNodePool = "nodepool" podHostZone = "zone" podHostArchitecture = "arch" @@ -60,7 +58,7 @@ var ( Namespace: "karpenter", Subsystem: "pods", Name: "state", - Help: "Pod state is the current state of pods. This metric can be used several ways as it is labeled by the pod name, namespace, owner, node, provisioner name, zone, architecture, capacity type, instance type and pod phase.", + Help: "Pod state is the current state of pods. This metric can be used several ways as it is labeled by the pod name, namespace, owner, node, nodepool name, zone, architecture, capacity type, instance type and pod phase.", }, labelNames(), ) @@ -94,7 +92,6 @@ func labelNames() []string { podNameSpace, ownerSelfLink, podHostName, - podProvisioner, podNodePool, podHostZone, podHostArchitecture, @@ -184,7 +181,6 @@ func (c *Controller) makeLabels(ctx context.Context, pod *v1.Pod) (prometheus.La metricLabels[podHostArchitecture] = node.Labels[v1.LabelArchStable] metricLabels[podHostCapacityType] = node.Labels[v1beta1.CapacityTypeLabelKey] metricLabels[podHostInstanceType] = node.Labels[v1.LabelInstanceTypeStable] - metricLabels[podProvisioner] = node.Labels[v1alpha5.ProvisionerNameLabelKey] metricLabels[podNodePool] = node.Labels[v1beta1.NodePoolLabelKey] return metricLabels, nil } diff --git a/pkg/controllers/node/termination/controller.go b/pkg/controllers/node/termination/controller.go index 587682f552..ccec5eba4c 100644 --- a/pkg/controllers/node/termination/controller.go +++ b/pkg/controllers/node/termination/controller.go @@ -86,12 +86,12 @@ func (c *Controller) Finalize(ctx context.Context, node *v1.Node) (reconcile.Res return reconcile.Result{}, fmt.Errorf("draining node, %w", err) } c.recorder.Publish(terminatorevents.NodeFailedToDrain(node, err)) - // If the underlying machine no longer exists. + // If the underlying nodeclaim no longer exists. if _, err := c.cloudProvider.Get(ctx, node.Spec.ProviderID); err != nil { if cloudprovider.IsNodeClaimNotFoundError(err) { return reconcile.Result{}, c.removeFinalizer(ctx, node) } - return reconcile.Result{}, fmt.Errorf("getting machine, %w", err) + return reconcile.Result{}, fmt.Errorf("getting nodeclaim, %w", err) } return reconcile.Result{RequeueAfter: 1 * time.Second}, nil } diff --git a/pkg/controllers/node/termination/terminator/terminator.go b/pkg/controllers/node/termination/terminator/terminator.go index b4848d2c80..a6297a8136 100644 --- a/pkg/controllers/node/termination/terminator/terminator.go +++ b/pkg/controllers/node/termination/terminator/terminator.go @@ -26,7 +26,6 @@ import ( "knative.dev/pkg/logging" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/karpenter/pkg/apis/v1alpha5" "sigs.k8s.io/karpenter/pkg/apis/v1beta1" podutil "sigs.k8s.io/karpenter/pkg/utils/pod" ) @@ -84,13 +83,11 @@ func (t *Terminator) Drain(ctx context.Context, node *v1.Node) error { return fmt.Errorf("listing pods on node, %w", err) } - _, isMachine := node.Labels[v1alpha5.ProvisionerNameLabelKey] // Skip node due to pods that are not able to be evicted podsToEvict := lo.FilterMap(pods.Items, func(po v1.Pod, _ int) (*v1.Pod, bool) { p := lo.ToPtr(po) - // Ignore pods that tolerate the node.kubernetes.io/unschedulable taint if linked to a machine - // or pods that tolerate the karpenter.sh/disruption taint if linked to a nodeclaim. - if lo.Ternary(isMachine, podutil.ToleratesUnschedulableTaint(p), podutil.ToleratesDisruptionNoScheduleTaint(p)) || + // Ignore pods that tolerate the karpenter.sh/disruption taint. + if podutil.ToleratesDisruptionNoScheduleTaint(p) || // Ignore static mirror pods podutil.IsOwnedByNode(p) || // Ignore if the pod is complete and doesn't need to be evicted diff --git a/pkg/controllers/nodeclaim/disruption/controller.go b/pkg/controllers/nodeclaim/disruption/controller.go index 88cf88166d..22e59db7f0 100644 --- a/pkg/controllers/nodeclaim/disruption/controller.go +++ b/pkg/controllers/nodeclaim/disruption/controller.go @@ -46,7 +46,7 @@ type nodeClaimReconciler interface { Reconcile(context.Context, *v1beta1.NodePool, *v1beta1.NodeClaim) (reconcile.Result, error) } -// Controller is a disruption controller that adds StatusConditions to Machines when they meet certain disruption conditions +// Controller is a disruption controller that adds StatusConditions to nodeclaims when they meet certain disruption conditions // e.g. When the NodeClaim has surpassed its owning provisioner's expirationTTL, then it is marked as "Expired" in the StatusConditions type Controller struct { kubeClient client.Client @@ -56,7 +56,7 @@ type Controller struct { emptiness *Emptiness } -// NewController constructs a machine disruption controller +// NewController constructs a nodeclaim disruption controller func NewController(clk clock.Clock, kubeClient client.Client, cluster *state.Cluster, cloudProvider cloudprovider.CloudProvider) operatorcontroller.Controller { return operatorcontroller.Typed[*v1beta1.NodeClaim](kubeClient, &Controller{ kubeClient: kubeClient, diff --git a/pkg/controllers/nodeclaim/disruption/drift.go b/pkg/controllers/nodeclaim/disruption/drift.go index bc956a525f..df5f5bdbd8 100644 --- a/pkg/controllers/nodeclaim/disruption/drift.go +++ b/pkg/controllers/nodeclaim/disruption/drift.go @@ -35,12 +35,11 @@ import ( ) const ( - ProvisionerDrifted cloudprovider.DriftReason = "ProvisionerDrifted" NodePoolDrifted cloudprovider.DriftReason = "NodePoolDrifted" RequirementsDrifted cloudprovider.DriftReason = "RequirementsDrifted" ) -// Drift is a machine sub-controller that adds or removes status conditions on drifted machines +// Drift is a nodeclaim sub-controller that adds or removes status conditions on drifted nodeclaims type Drift struct { cloudProvider cloudprovider.CloudProvider } @@ -120,11 +119,11 @@ func areStaticFieldsDrifted(nodePool *v1beta1.NodePool, nodeClaim *v1beta1.NodeC } func areRequirementsDrifted(nodePool *v1beta1.NodePool, nodeClaim *v1beta1.NodeClaim) cloudprovider.DriftReason { - provisionerReq := scheduling.NewNodeSelectorRequirements(nodePool.Spec.Template.Spec.Requirements...) + nodepoolReq := scheduling.NewNodeSelectorRequirements(nodePool.Spec.Template.Spec.Requirements...) nodeClaimReq := scheduling.NewLabelRequirements(nodeClaim.Labels) - // Every provisioner requirement is compatible with the NodeClaim label set - if nodeClaimReq.Compatible(provisionerReq) != nil { + // Every nodepool requirement is compatible with the NodeClaim label set + if nodeClaimReq.Compatible(nodepoolReq) != nil { return RequirementsDrifted } diff --git a/pkg/controllers/nodeclaim/disruption/drift_test.go b/pkg/controllers/nodeclaim/disruption/drift_test.go index c9416b5907..195935651f 100644 --- a/pkg/controllers/nodeclaim/disruption/drift_test.go +++ b/pkg/controllers/nodeclaim/disruption/drift_test.go @@ -157,9 +157,9 @@ var _ = Describe("Drift", func() { }) Context("NodeRequirement Drift", func() { DescribeTable("", - func(oldProvisionerReq []v1.NodeSelectorRequirement, newProvisionerReq []v1.NodeSelectorRequirement, labels map[string]string, drifted bool) { + func(oldNodePoolReq []v1.NodeSelectorRequirement, newNodePoolReq []v1.NodeSelectorRequirement, labels map[string]string, drifted bool) { cp.Drifted = "" - nodePool.Spec.Template.Spec.Requirements = oldProvisionerReq + nodePool.Spec.Template.Spec.Requirements = oldNodePoolReq nodeClaim.Labels = lo.Assign(nodeClaim.Labels, labels) ExpectApplied(ctx, env.Client, nodePool, nodeClaim) @@ -167,7 +167,7 @@ var _ = Describe("Drift", func() { nodeClaim = ExpectExists(ctx, env.Client, nodeClaim) Expect(nodeClaim.StatusConditions().GetCondition(v1beta1.Drifted)).To(BeNil()) - nodePool.Spec.Template.Spec.Requirements = newProvisionerReq + nodePool.Spec.Template.Spec.Requirements = newNodePoolReq ExpectApplied(ctx, env.Client, nodePool) ExpectReconcileSucceeded(ctx, nodeClaimDisruptionController, client.ObjectKeyFromObject(nodeClaim)) nodeClaim = ExpectExists(ctx, env.Client, nodeClaim) @@ -360,7 +360,7 @@ var _ = Describe("Drift", func() { }) }) - Context("Provisioner Static Drift", func() { + Context("NodePool Static Drift", func() { var nodePoolOptions v1beta1.NodePool var nodePoolController controller.Controller BeforeEach(func() { diff --git a/pkg/controllers/nodeclaim/disruption/emptiness.go b/pkg/controllers/nodeclaim/disruption/emptiness.go index bd87ee95c5..6bbaf1f918 100644 --- a/pkg/controllers/nodeclaim/disruption/emptiness.go +++ b/pkg/controllers/nodeclaim/disruption/emptiness.go @@ -33,7 +33,7 @@ import ( nodeclaimutil "sigs.k8s.io/karpenter/pkg/utils/nodeclaim" ) -// Emptiness is a machine sub-controller that adds or removes status conditions on empty machines based on TTLSecondsAfterEmpty +// Emptiness is a nodeclaim sub-controller that adds or removes status conditions on empty nodeclaims based on TTLSecondsAfterEmpty type Emptiness struct { kubeClient client.Client cluster *state.Cluster diff --git a/pkg/controllers/nodeclaim/disruption/expiration.go b/pkg/controllers/nodeclaim/disruption/expiration.go index 0c6d9b0eea..e1ce918ade 100644 --- a/pkg/controllers/nodeclaim/disruption/expiration.go +++ b/pkg/controllers/nodeclaim/disruption/expiration.go @@ -30,7 +30,7 @@ import ( nodeclaimutil "sigs.k8s.io/karpenter/pkg/utils/nodeclaim" ) -// Expiration is a machine sub-controller that adds or removes status conditions on expired machines based on TTLSecondsUntilExpired +// Expiration is a nodeclaim sub-controller that adds or removes status conditions on expired nodeclaims based on TTLSecondsUntilExpired type Expiration struct { kubeClient client.Client clock clock.Clock @@ -53,12 +53,10 @@ func (e *Expiration) Reconcile(ctx context.Context, nodePool *v1beta1.NodePool, if nodeclaimutil.IgnoreNodeNotFoundError(nodeclaimutil.IgnoreDuplicateNodeError(err)) != nil { return reconcile.Result{}, err } - // We do the expiration check in this way since there is still a migration path for creating Machines from Nodes + // We do the expiration check in this way since there is still a migration path for creating nodeclaims from Nodes // In this case, we need to make sure that we take the older of the two for expiration - // TODO @joinnis: This check that takes the minimum between the Node and Machine CreationTimestamps can be removed - // once machine migration is ripped out, which should happen when apis and Karpenter are promoted to v1 var expirationTime time.Time - if node == nil || nodeClaim.CreationTimestamp.Before(&node.CreationTimestamp) { + if node == nil { expirationTime = nodeClaim.CreationTimestamp.Add(*nodePool.Spec.Disruption.ExpireAfter.Duration) } else { expirationTime = node.CreationTimestamp.Add(*nodePool.Spec.Disruption.ExpireAfter.Duration) diff --git a/pkg/controllers/nodeclaim/lifecycle/controller.go b/pkg/controllers/nodeclaim/lifecycle/controller.go index 3f17f58f7a..462fd33547 100644 --- a/pkg/controllers/nodeclaim/lifecycle/controller.go +++ b/pkg/controllers/nodeclaim/lifecycle/controller.go @@ -52,7 +52,7 @@ type nodeClaimReconciler interface { // Controller is a NodeClaim Lifecycle controller that manages the lifecycle of the NodeClaim up until its termination // The controller is responsible for ensuring that new Nodes get launched, that they have properly registered with -// the cluster as nodes and that they are properly initialized, ensuring that Machines that do not have matching nodes +// the cluster as nodes and that they are properly initialized, ensuring that nodeclaims that do not have matching nodes // after some liveness TTL are removed type Controller struct { kubeClient client.Client diff --git a/pkg/controllers/nodeclaim/lifecycle/initialization.go b/pkg/controllers/nodeclaim/lifecycle/initialization.go index a03d26ce9b..e3b17ad8b2 100644 --- a/pkg/controllers/nodeclaim/lifecycle/initialization.go +++ b/pkg/controllers/nodeclaim/lifecycle/initialization.go @@ -40,7 +40,7 @@ type Initialization struct { // a) its current status is set to Ready // b) all the startup taints have been removed from the node // c) all extended resources have been registered -// This method handles both nil provisioners and nodes without extended resources gracefully. +// This method handles both nil nodepools and nodes without extended resources gracefully. func (i *Initialization) Reconcile(ctx context.Context, nodeClaim *v1beta1.NodeClaim) (reconcile.Result, error) { if nodeClaim.StatusConditions().GetCondition(v1beta1.Initialized).IsTrue() { return reconcile.Result{}, nil @@ -98,7 +98,7 @@ func KnownEphemeralTaintsRemoved(node *v1.Node) (*v1.Taint, bool) { return nil, true } -// StartupTaintsRemoved returns true if there are no startup taints registered for the provisioner, or if all startup +// StartupTaintsRemoved returns true if there are no startup taints registered for the nodepool, or if all startup // taints have been removed from the node func StartupTaintsRemoved(node *v1.Node, nodeClaim *v1beta1.NodeClaim) (*v1.Taint, bool) { if nodeClaim != nil { diff --git a/pkg/controllers/nodeclaim/lifecycle/launch_test.go b/pkg/controllers/nodeclaim/lifecycle/launch_test.go index e3fda79342..8c0e5d1319 100644 --- a/pkg/controllers/nodeclaim/lifecycle/launch_test.go +++ b/pkg/controllers/nodeclaim/lifecycle/launch_test.go @@ -21,7 +21,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/karpenter/pkg/apis/v1alpha5" "sigs.k8s.io/karpenter/pkg/apis/v1beta1" "sigs.k8s.io/karpenter/pkg/cloudprovider" "sigs.k8s.io/karpenter/pkg/test" @@ -55,11 +54,11 @@ var _ = Describe("Launch", func() { _, err := cloudProvider.Get(ctx, nodeClaim.Status.ProviderID) Expect(err).ToNot(HaveOccurred()) }) - It("should add the MachineLaunched status condition after creating the NodeClaim", func() { + It("should add the Launched status condition after creating the NodeClaim", func() { nodeClaim := test.NodeClaim(v1beta1.NodeClaim{ ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{ - v1alpha5.ProvisionerNameLabelKey: nodePool.Name, + v1beta1.NodePoolLabelKey: nodePool.Name, }, }, }) @@ -69,7 +68,7 @@ var _ = Describe("Launch", func() { nodeClaim = ExpectExists(ctx, env.Client, nodeClaim) Expect(ExpectStatusConditionExists(nodeClaim, v1beta1.Launched).Status).To(Equal(v1.ConditionTrue)) }) - It("should delete the machine if InsufficientCapacity is returned from the cloudprovider", func() { + It("should delete the nodeclaim if InsufficientCapacity is returned from the cloudprovider", func() { cloudProvider.NextCreateErr = cloudprovider.NewInsufficientCapacityError(fmt.Errorf("all instance types were unavailable")) nodeClaim := test.NodeClaim() ExpectApplied(ctx, env.Client, nodeClaim) diff --git a/pkg/controllers/nodeclaim/termination/controller.go b/pkg/controllers/nodeclaim/termination/controller.go index 467b79ef83..f7c2bd4814 100644 --- a/pkg/controllers/nodeclaim/termination/controller.go +++ b/pkg/controllers/nodeclaim/termination/controller.go @@ -34,7 +34,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" - "sigs.k8s.io/karpenter/pkg/apis/v1alpha5" "sigs.k8s.io/karpenter/pkg/apis/v1beta1" "sigs.k8s.io/karpenter/pkg/cloudprovider" operatorcontroller "sigs.k8s.io/karpenter/pkg/operator/controller" @@ -82,7 +81,7 @@ func (c *Controller) Finalize(ctx context.Context, nodeClaim *v1beta1.NodeClaim) if len(nodes) > 0 { return reconcile.Result{}, nil } - if nodeClaim.Status.ProviderID != "" || nodeClaim.Annotations[v1alpha5.MachineLinkedAnnotationKey] != "" { + if nodeClaim.Status.ProviderID != "" { if err = c.cloudProvider.Delete(ctx, nodeClaim); cloudprovider.IgnoreNodeClaimNotFoundError(err) != nil { return reconcile.Result{}, fmt.Errorf("terminating cloudprovider instance, %w", err) } diff --git a/pkg/controllers/nodepool/counter/controller.go b/pkg/controllers/nodepool/counter/controller.go index c81b0707b8..06bccfb0ee 100644 --- a/pkg/controllers/nodepool/counter/controller.go +++ b/pkg/controllers/nodepool/counter/controller.go @@ -58,13 +58,13 @@ func NewController(kubeClient client.Client, cluster *state.Cluster) operatorcon // Reconcile a control loop for the resource func (c *Controller) Reconcile(ctx context.Context, nodePool *v1beta1.NodePool) (reconcile.Result, error) { // We need to ensure that our internal cluster state mechanism is synced before we proceed - // Otherwise, we have the potential to patch over the status with a lower value for the provisioner resource + // Otherwise, we have the potential to patch over the status with a lower value for the nodepool resource // counts on startup if !c.cluster.Synced(ctx) { return reconcile.Result{RequeueAfter: time.Second}, nil } stored := nodePool.DeepCopy() - // Determine resource usage and update provisioner.status.resources + // Determine resource usage and update nodepool.status.resources nodePool.Status.Resources = c.resourceCountsFor(v1beta1.NodePoolLabelKey, nodePool.Name) if !equality.Semantic.DeepEqual(stored, nodePool) { if err := nodepoolutil.PatchStatus(ctx, c.kubeClient, stored, nodePool); err != nil { @@ -76,8 +76,8 @@ func (c *Controller) Reconcile(ctx context.Context, nodePool *v1beta1.NodePool) func (c *Controller) resourceCountsFor(ownerLabel string, ownerName string) v1.ResourceList { var res v1.ResourceList - // Record all resources provisioned by the provisioners, we look at the cluster state nodes as their capacity - // is accurately reported even for nodes that haven't fully started yet. This allows us to update our provisioner + // Record all resources provisioned by the nodepools, we look at the cluster state nodes as their capacity + // is accurately reported even for nodes that haven't fully started yet. This allows us to update our nodepool // status immediately upon node creation instead of waiting for the node to become ready. c.cluster.ForEachNode(func(n *state.StateNode) bool { // Don't count nodes that we are planning to delete. This is to ensure that we are consistent throughout diff --git a/pkg/controllers/provisioning/provisioner.go b/pkg/controllers/provisioning/provisioner.go index 873024dedc..46edb1d52f 100644 --- a/pkg/controllers/provisioning/provisioner.go +++ b/pkg/controllers/provisioning/provisioner.go @@ -196,7 +196,7 @@ func (p *Provisioner) consolidationWarnings(ctx context.Context, po v1.Pod) { } } -var ErrNodePoolsNotFound = errors.New("no nodepools or provisioners found") +var ErrNodePoolsNotFound = errors.New("no nodepools found") //nolint:gocyclo func (p *Provisioner) NewScheduler(ctx context.Context, pods []*v1.Pod, stateNodes []*state.StateNode, opts scheduler.SchedulerOptions) (*scheduler.Scheduler, error) { diff --git a/pkg/controllers/provisioning/suite_test.go b/pkg/controllers/provisioning/suite_test.go index d4c3b8b897..246549e868 100644 --- a/pkg/controllers/provisioning/suite_test.go +++ b/pkg/controllers/provisioning/suite_test.go @@ -394,7 +394,7 @@ var _ = Describe("Provisioning", func() { // only available instance type has 2 GPUs which would exceed the limit ExpectNotScheduled(ctx, env.Client, pod) }) - It("should not schedule to a provisioner after a scheduling round if limits would be exceeded", func() { + It("should not schedule to a nodepool after a scheduling round if limits would be exceeded", func() { ExpectApplied(ctx, env.Client, test.NodePool(v1beta1.NodePool{ Spec: v1beta1.NodePoolSpec{ Limits: v1beta1.Limits(v1.ResourceList{v1.ResourceCPU: resource.MustParse("2")}), @@ -1552,8 +1552,8 @@ var _ = Describe("Provisioning", func() { node := ExpectScheduled(ctx, env.Client, pod) Expect(node.Labels[v1beta1.NodePoolLabelKey]).ToNot(Equal(nodePool.Name)) }) - Context("Weighted nodePools", func() { - It("should schedule to the provisioner with the highest priority always", func() { + Context("Weighted NodePools", func() { + It("should schedule to the nodepool with the highest priority always", func() { nodePools := []client.Object{ test.NodePool(), test.NodePool(v1beta1.NodePool{Spec: v1beta1.NodePoolSpec{Weight: ptr.Int32(20)}}), @@ -1569,7 +1569,7 @@ var _ = Describe("Provisioning", func() { Expect(node.Labels[v1beta1.NodePoolLabelKey]).To(Equal(nodePools[2].GetName())) } }) - It("should schedule to explicitly selected provisioner even if other nodePools are higher priority", func() { + It("should schedule to explicitly selected nodepool even if other nodePools are higher priority", func() { targetedNodePool := test.NodePool() nodePools := []client.Object{ targetedNodePool, diff --git a/pkg/controllers/state/cluster.go b/pkg/controllers/state/cluster.go index a145fdb56e..5ebee5f67a 100644 --- a/pkg/controllers/state/cluster.go +++ b/pkg/controllers/state/cluster.go @@ -59,8 +59,8 @@ type Cluster struct { // A monotonically increasing timestamp representing the time state of the // cluster with respect to consolidation. This increases when something has // changed about the cluster that might make consolidation possible. By recording - // the state, interested deprovisioners can check to see if this has changed to - // optimize and not try to deprovision if nothing about the cluster has changed. + // the state, interested disruption methods can check to see if this has changed to + // optimize and not try to disrupt if nothing about the cluster has changed. clusterState time.Time antiAffinityPods sync.Map // pod namespaced name -> *v1.Pod of pods that have required anti affinities } @@ -220,7 +220,7 @@ func (c *Cluster) UpdateNodeClaim(nodeClaim *v1beta1.NodeClaim) { defer c.mu.Unlock() if nodeClaim.Status.ProviderID == "" { - return // We can't reconcile machines that don't yet have provider ids + return // We can't reconcile nodeclaims that don't yet have provider ids } n := c.newStateFromNodeClaim(nodeClaim, c.nodes[nodeClaim.Status.ProviderID]) c.nodes[nodeClaim.Status.ProviderID] = n @@ -377,21 +377,16 @@ func (c *Cluster) newStateFromNodeClaim(nodeClaim *v1beta1.NodeClaim, oldNode *S oldNode = NewNode() } n := &StateNode{ - Node: oldNode.Node, - NodeClaim: nodeClaim, - inflightInitialized: oldNode.inflightInitialized, - inflightAllocatable: oldNode.inflightAllocatable, - inflightCapacity: oldNode.inflightCapacity, - startupTaintsInitialized: oldNode.startupTaintsInitialized, - startupTaints: oldNode.startupTaints, - daemonSetRequests: oldNode.daemonSetRequests, - daemonSetLimits: oldNode.daemonSetLimits, - podRequests: oldNode.podRequests, - podLimits: oldNode.podLimits, - hostPortUsage: oldNode.hostPortUsage, - volumeUsage: oldNode.volumeUsage, - markedForDeletion: oldNode.markedForDeletion, - nominatedUntil: oldNode.nominatedUntil, + Node: oldNode.Node, + NodeClaim: nodeClaim, + daemonSetRequests: oldNode.daemonSetRequests, + daemonSetLimits: oldNode.daemonSetLimits, + podRequests: oldNode.podRequests, + podLimits: oldNode.podLimits, + hostPortUsage: oldNode.hostPortUsage, + volumeUsage: oldNode.volumeUsage, + markedForDeletion: oldNode.markedForDeletion, + nominatedUntil: oldNode.nominatedUntil, } // Cleanup the old nodeClaim with its old providerID if its providerID changes // This can happen since nodes don't get created with providerIDs. Rather, CCM picks up the @@ -420,25 +415,18 @@ func (c *Cluster) newStateFromNode(ctx context.Context, node *v1.Node, oldNode * oldNode = NewNode() } n := &StateNode{ - Node: node, - NodeClaim: oldNode.NodeClaim, - inflightInitialized: oldNode.inflightInitialized, - inflightAllocatable: oldNode.inflightAllocatable, - inflightCapacity: oldNode.inflightCapacity, - startupTaintsInitialized: oldNode.startupTaintsInitialized, - startupTaints: oldNode.startupTaints, - daemonSetRequests: map[types.NamespacedName]v1.ResourceList{}, - daemonSetLimits: map[types.NamespacedName]v1.ResourceList{}, - podRequests: map[types.NamespacedName]v1.ResourceList{}, - podLimits: map[types.NamespacedName]v1.ResourceList{}, - hostPortUsage: scheduling.NewHostPortUsage(), - volumeUsage: scheduling.NewVolumeUsage(), - markedForDeletion: oldNode.markedForDeletion, - nominatedUntil: oldNode.nominatedUntil, + Node: node, + NodeClaim: oldNode.NodeClaim, + daemonSetRequests: map[types.NamespacedName]v1.ResourceList{}, + daemonSetLimits: map[types.NamespacedName]v1.ResourceList{}, + podRequests: map[types.NamespacedName]v1.ResourceList{}, + podLimits: map[types.NamespacedName]v1.ResourceList{}, + hostPortUsage: scheduling.NewHostPortUsage(), + volumeUsage: scheduling.NewVolumeUsage(), + markedForDeletion: oldNode.markedForDeletion, + nominatedUntil: oldNode.nominatedUntil, } if err := multierr.Combine( - c.populateStartupTaints(ctx, n), - c.populateInflight(ctx, n), c.populateResourceRequests(ctx, n), c.populateVolumeLimits(ctx, n), ); err != nil { @@ -466,61 +454,6 @@ func (c *Cluster) cleanupNode(name string) { } } -func (c *Cluster) populateStartupTaints(ctx context.Context, n *StateNode) error { - // We only need to populate the startup taints once - if n.startupTaintsInitialized { - return nil - } - - nodePoolName, ok := n.Labels()[v1beta1.NodePoolLabelKey] - if !ok { - return nil - } - nodePool := &v1beta1.NodePool{} - if err := c.kubeClient.Get(ctx, types.NamespacedName{Name: nodePoolName}, nodePool); err != nil { - return client.IgnoreNotFound(err) - } - n.startupTaintsInitialized = true - n.startupTaints = nodePool.Spec.Template.Spec.StartupTaints - return nil -} - -func (c *Cluster) populateInflight(ctx context.Context, n *StateNode) error { - // We only need to set inflight details once. This prevents us from logging spurious errors when the cloud provider - // instance types change. - if n.inflightInitialized { - return nil - } - // If the node is already initialized, we don't need to populate its inflight capacity - // since its capacity is already represented by the node status - if n.Initialized() { - return nil - } - - nodePoolName, ok := n.Labels()[v1beta1.NodePoolLabelKey] - if !ok { - return nil - } - nodePool := &v1beta1.NodePool{} - if err := c.kubeClient.Get(ctx, types.NamespacedName{Name: nodePoolName}, nodePool); err != nil { - return client.IgnoreNotFound(err) - } - instanceTypes, err := c.cloudProvider.GetInstanceTypes(ctx, nodePool) - if err != nil { - return err - } - instanceType, ok := lo.Find(instanceTypes, func(it *cloudprovider.InstanceType) bool { - return it.Name == n.Labels()[v1.LabelInstanceTypeStable] - }) - if !ok { - return fmt.Errorf("instance type '%s' not found", n.Labels()[v1.LabelInstanceTypeStable]) - } - n.inflightInitialized = true - n.inflightCapacity = instanceType.Capacity - n.inflightAllocatable = instanceType.Allocatable() - return nil -} - func (c *Cluster) populateVolumeLimits(ctx context.Context, n *StateNode) error { var csiNode storagev1.CSINode if err := c.kubeClient.Get(ctx, client.ObjectKey{Name: n.Node.Name}, &csiNode); err != nil { diff --git a/pkg/controllers/state/informer/nodeclaim.go b/pkg/controllers/state/informer/nodeclaim.go index 546c296cd8..b6123b17e9 100644 --- a/pkg/controllers/state/informer/nodeclaim.go +++ b/pkg/controllers/state/informer/nodeclaim.go @@ -30,7 +30,7 @@ import ( operatorcontroller "sigs.k8s.io/karpenter/pkg/operator/controller" ) -// NodeClaimController reconciles machine for the purpose of maintaining state. +// NodeClaimController reconciles nodeclaim for the purpose of maintaining state. type NodeClaimController struct { kubeClient client.Client cluster *state.Cluster diff --git a/pkg/controllers/state/statenode.go b/pkg/controllers/state/statenode.go index 05ec154725..53f69e0c07 100644 --- a/pkg/controllers/state/statenode.go +++ b/pkg/controllers/state/statenode.go @@ -77,13 +77,6 @@ type StateNode struct { Node *v1.Node NodeClaim *v1beta1.NodeClaim - inflightInitialized bool // TODO @joinnis: This can be removed when machine is added - inflightAllocatable v1.ResourceList // TODO @joinnis: This can be removed when machine is added - inflightCapacity v1.ResourceList // TODO @joinnis: This can be removed when machine is added - - startupTaintsInitialized bool // TODO: @joinnis: This can be removed when machine is added - startupTaints []v1.Taint // TODO: @joinnis: This can be removed when machine is added - // daemonSetRequests is the total amount of resources that have been requested by daemon sets. This allows users // of the Node to identify the remaining resources that we expect future daemonsets to consume. daemonSetRequests map[types.NamespacedName]v1.ResourceList @@ -103,15 +96,12 @@ type StateNode struct { func NewNode() *StateNode { return &StateNode{ - inflightAllocatable: v1.ResourceList{}, - inflightCapacity: v1.ResourceList{}, - startupTaints: []v1.Taint{}, - daemonSetRequests: map[types.NamespacedName]v1.ResourceList{}, - daemonSetLimits: map[types.NamespacedName]v1.ResourceList{}, - podRequests: map[types.NamespacedName]v1.ResourceList{}, - podLimits: map[types.NamespacedName]v1.ResourceList{}, - hostPortUsage: scheduling.NewHostPortUsage(), - volumeUsage: scheduling.NewVolumeUsage(), + daemonSetRequests: map[types.NamespacedName]v1.ResourceList{}, + daemonSetLimits: map[types.NamespacedName]v1.ResourceList{}, + podRequests: map[types.NamespacedName]v1.ResourceList{}, + podLimits: map[types.NamespacedName]v1.ResourceList{}, + hostPortUsage: scheduling.NewHostPortUsage(), + volumeUsage: scheduling.NewVolumeUsage(), } } @@ -154,8 +144,8 @@ func (in *StateNode) HostName() string { } func (in *StateNode) Annotations() map[string]string { - // If the machine exists and the state node isn't initialized - // use the machine representation of the annotations + // If the nodeclaim exists and the state node isn't initialized + // use the nodeclaim representation of the annotations if in.Node == nil { return in.NodeClaim.Annotations } @@ -174,8 +164,8 @@ func (in *StateNode) GetLabels() map[string]string { } func (in *StateNode) Labels() map[string]string { - // If the machine exists and the state node isn't registered - // use the machine representation of the labels + // If the nodeclaim exists and the state node isn't registered + // use the nodeclaim representation of the labels if in.Node == nil { return in.NodeClaim.Labels } @@ -193,12 +183,8 @@ func (in *StateNode) Taints() []v1.Taint { // re-appears on the node for a different reason (e.g. the node is cordoned) we will assume that pods can // schedule against the node in the future incorrectly. ephemeralTaints := scheduling.KnownEphemeralTaints - if !in.Initialized() && in.Managed() { - if in.NodeClaim != nil { - ephemeralTaints = append(ephemeralTaints, in.NodeClaim.Spec.StartupTaints...) - } else { - ephemeralTaints = append(ephemeralTaints, in.startupTaints...) - } + if !in.Initialized() && in.Managed() && in.NodeClaim != nil { + ephemeralTaints = append(ephemeralTaints, in.NodeClaim.Spec.StartupTaints...) } var taints []v1.Taint @@ -247,17 +233,6 @@ func (in *StateNode) Capacity() v1.ResourceList { } return in.NodeClaim.Status.Capacity } - // TODO @joinnis: Remove this when machine migration is complete - if !in.Initialized() && in.Managed() { - // Override any zero quantity values in the node status - ret := lo.Assign(in.Node.Status.Capacity) - for resourceName, quantity := range in.inflightCapacity { - if resources.IsZero(ret[resourceName]) { - ret[resourceName] = quantity - } - } - return ret - } return in.Node.Status.Capacity } @@ -275,17 +250,6 @@ func (in *StateNode) Allocatable() v1.ResourceList { } return in.NodeClaim.Status.Allocatable } - // TODO @joinnis: Remove this when machine migration is complete - if !in.Initialized() && in.Managed() { - // Override any zero quantity values in the node status - ret := lo.Assign(in.Node.Status.Allocatable) - for resourceName, quantity := range in.inflightAllocatable { - if resources.IsZero(ret[resourceName]) { - ret[resourceName] = quantity - } - } - return ret - } return in.Node.Status.Allocatable } diff --git a/pkg/controllers/state/suite_test.go b/pkg/controllers/state/suite_test.go index 8bb00fdc0e..c36cf74101 100644 --- a/pkg/controllers/state/suite_test.go +++ b/pkg/controllers/state/suite_test.go @@ -28,7 +28,6 @@ import ( "knative.dev/pkg/ptr" "sigs.k8s.io/karpenter/pkg/apis" - "sigs.k8s.io/karpenter/pkg/apis/v1alpha5" "sigs.k8s.io/karpenter/pkg/apis/v1beta1" "sigs.k8s.io/karpenter/pkg/cloudprovider/fake" "sigs.k8s.io/karpenter/pkg/controllers/state/informer" @@ -182,7 +181,7 @@ var _ = Describe("Volume Usage/Limits", func() { csiProvider: sets.New("test"), })).ToNot(BeNil()) - // Reconcile the machine one more time to ensure that we maintain our volume usage state + // Reconcile the nodeclaim one more time to ensure that we maintain our volume usage state ExpectReconcileSucceeded(ctx, nodeClaimController, client.ObjectKeyFromObject(nodeClaim)) // Ensure that we still consider adding another volume to the node breaching our volume limits @@ -889,7 +888,7 @@ var _ = Describe("Node Resource Level", func() { v1beta1.NodePoolLabelKey: nodePool.Name, v1.LabelInstanceTypeStable: cloudProvider.InstanceTypes[0].Name, }, - Finalizers: []string{v1alpha5.TerminationFinalizer}, + Finalizers: []string{v1beta1.TerminationFinalizer}, }, Allocatable: map[v1.ResourceName]resource.Quantity{ v1.ResourceCPU: resource.MustParse("4"), diff --git a/pkg/controllers/state/zz_generated.deepcopy.go b/pkg/controllers/state/zz_generated.deepcopy.go index 0406cd6cd2..8c38d12b7b 100644 --- a/pkg/controllers/state/zz_generated.deepcopy.go +++ b/pkg/controllers/state/zz_generated.deepcopy.go @@ -39,27 +39,6 @@ func (in *StateNode) DeepCopyInto(out *StateNode) { *out = new(v1beta1.NodeClaim) (*in).DeepCopyInto(*out) } - if in.inflightAllocatable != nil { - in, out := &in.inflightAllocatable, &out.inflightAllocatable - *out = make(v1.ResourceList, len(*in)) - for key, val := range *in { - (*out)[key] = val.DeepCopy() - } - } - if in.inflightCapacity != nil { - in, out := &in.inflightCapacity, &out.inflightCapacity - *out = make(v1.ResourceList, len(*in)) - for key, val := range *in { - (*out)[key] = val.DeepCopy() - } - } - if in.startupTaints != nil { - in, out := &in.startupTaints, &out.startupTaints - *out = make([]v1.Taint, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } if in.daemonSetRequests != nil { in, out := &in.daemonSetRequests, &out.daemonSetRequests *out = make(map[types.NamespacedName]v1.ResourceList, len(*in)) diff --git a/pkg/metrics/constants.go b/pkg/metrics/constants.go index 8dbc25dca4..6dab997282 100644 --- a/pkg/metrics/constants.go +++ b/pkg/metrics/constants.go @@ -24,10 +24,9 @@ const ( // Common namespace for application metrics. Namespace = "karpenter" - ProvisionerLabel = "provisioner" - NodePoolLabel = "nodepool" - ReasonLabel = "reason" - TypeLabel = "type" + NodePoolLabel = "nodepool" + ReasonLabel = "reason" + TypeLabel = "type" // Reasons for CREATE/DELETE shared metrics ConsolidationReason = "consolidation" diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index ea6fa4cc8b..b82ec3433d 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -111,7 +111,7 @@ var ( Namespace: Namespace, Subsystem: NodeSubsystem, Name: "created", - Help: "Number of nodes created in total by Karpenter. Labeled by owning provisioner.", + Help: "Number of nodes created in total by Karpenter. Labeled by owning nodepool.", }, []string{ NodePoolLabel, @@ -122,7 +122,7 @@ var ( Namespace: Namespace, Subsystem: NodeSubsystem, Name: "terminated", - Help: "Number of nodes terminated in total by Karpenter. Labeled by owning provisioner.", + Help: "Number of nodes terminated in total by Karpenter. Labeled by owning nodepool.", }, []string{ NodePoolLabel, diff --git a/pkg/operator/controller/suite_test.go b/pkg/operator/controller/suite_test.go index c1015a00b3..b4228afbe9 100644 --- a/pkg/operator/controller/suite_test.go +++ b/pkg/operator/controller/suite_test.go @@ -27,7 +27,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/karpenter/pkg/apis" - "sigs.k8s.io/karpenter/pkg/apis/v1alpha5" + "sigs.k8s.io/karpenter/pkg/apis/v1beta1" "sigs.k8s.io/karpenter/pkg/operator/controller" "sigs.k8s.io/karpenter/pkg/operator/scheme" "sigs.k8s.io/karpenter/pkg/test" @@ -76,7 +76,7 @@ var _ = Describe("Typed", func() { node := test.Node(test.NodeOptions{ ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{ - v1alpha5.ProvisionerNameLabelKey: "default", + v1beta1.NodePoolLabelKey: "default", }, }, }) @@ -85,7 +85,7 @@ var _ = Describe("Typed", func() { ReconcileAssertions: []TypedReconcileAssertion[*v1.Node]{ func(ctx context.Context, n *v1.Node) { Expect(n.Name).To(Equal(node.Name)) - Expect(n.Labels).To(HaveKeyWithValue(v1alpha5.ProvisionerNameLabelKey, "default")) + Expect(n.Labels).To(HaveKeyWithValue(v1beta1.NodePoolLabelKey, "default")) }, }, } @@ -96,7 +96,7 @@ var _ = Describe("Typed", func() { node := test.Node(test.NodeOptions{ ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{ - v1alpha5.ProvisionerNameLabelKey: "default", + v1beta1.NodePoolLabelKey: "default", }, Finalizers: []string{ "testing/finalizer", diff --git a/pkg/scheduling/requirements.go b/pkg/scheduling/requirements.go index 3bb02c89e3..30e3b35820 100644 --- a/pkg/scheduling/requirements.go +++ b/pkg/scheduling/requirements.go @@ -25,7 +25,6 @@ import ( v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/sets" - "sigs.k8s.io/karpenter/pkg/apis/v1alpha5" "sigs.k8s.io/karpenter/pkg/apis/v1beta1" "sigs.k8s.io/karpenter/pkg/utils/functional" ) @@ -259,7 +258,7 @@ func (r Requirements) Intersects(requirements Requirements) (errs error) { func (r Requirements) Labels() map[string]string { labels := map[string]string{} for key, requirement := range r { - if !v1alpha5.IsRestrictedNodeLabel(key) && !v1beta1.IsRestrictedNodeLabel(key) { + if !v1beta1.IsRestrictedNodeLabel(key) { if value := requirement.Any(); value != "" { labels[key] = value } @@ -270,7 +269,7 @@ func (r Requirements) Labels() map[string]string { func (r Requirements) String() string { requirements := lo.Reject(r.Values(), func(requirement *Requirement, _ int) bool { - return v1alpha5.RestrictedLabels.Has(requirement.Key) || v1beta1.RestrictedLabels.Has(requirement.Key) + return v1beta1.RestrictedLabels.Has(requirement.Key) }) stringRequirements := lo.Map(requirements, func(requirement *Requirement, _ int) string { return requirement.String() }) slices.Sort(stringRequirements) diff --git a/pkg/test/machine.go b/pkg/test/machine.go deleted file mode 100644 index 14f7c1bb7a..0000000000 --- a/pkg/test/machine.go +++ /dev/null @@ -1,65 +0,0 @@ -/* -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package test - -import ( - "fmt" - - "github.com/imdario/mergo" - v1 "k8s.io/api/core/v1" - - "sigs.k8s.io/karpenter/pkg/apis/v1alpha5" -) - -// Machine creates a test machine with defaults that can be overridden by MachineOptions. -// Overrides are applied in order, with a last write wins semantic. -func Machine(overrides ...v1alpha5.Machine) *v1alpha5.Machine { - override := v1alpha5.Machine{} - for _, opts := range overrides { - if err := mergo.Merge(&override, opts, mergo.WithOverride); err != nil { - panic(fmt.Sprintf("failed to merge: %v", err)) - } - } - if override.Name == "" { - override.Name = RandomName() - } - if override.Status.ProviderID == "" { - override.Status.ProviderID = RandomProviderID() - } - return &v1alpha5.Machine{ - ObjectMeta: ObjectMeta(override.ObjectMeta), - Spec: override.Spec, - Status: override.Status, - } -} - -func MachineAndNode(overrides ...v1alpha5.Machine) (*v1alpha5.Machine, *v1.Node) { - m := Machine(overrides...) - return m, MachineLinkedNode(m) -} - -// MachinesAndNodes creates homogeneous groups of machines and nodes based on the passed in options, evenly divided by the total machines requested -func MachinesAndNodes(total int, options ...v1alpha5.Machine) ([]*v1alpha5.Machine, []*v1.Node) { - machines := make([]*v1alpha5.Machine, total) - nodes := make([]*v1.Node, total) - for _, opts := range options { - for i := 0; i < total/len(options); i++ { - machine, node := MachineAndNode(opts) - machines[i] = machine - nodes[i] = node - } - } - return machines, nodes -} diff --git a/pkg/test/nodeclaim.go b/pkg/test/nodeclaim.go index 0d7c008caa..c22e3b63e5 100644 --- a/pkg/test/nodeclaim.go +++ b/pkg/test/nodeclaim.go @@ -58,7 +58,7 @@ func NodeClaimAndNode(overrides ...v1beta1.NodeClaim) (*v1beta1.NodeClaim, *v1.N return nc, NodeClaimLinkedNode(nc) } -// NodeClaimsAndNodes creates homogeneous groups of NodeClaims and Nodes based on the passed in options, evenly divided by the total machines requested +// NodeClaimsAndNodes creates homogeneous groups of NodeClaims and Nodes based on the passed in options, evenly divided by the total nodeclaims requested func NodeClaimsAndNodes(total int, options ...v1beta1.NodeClaim) ([]*v1beta1.NodeClaim, []*v1.Node) { nodeClaims := make([]*v1beta1.NodeClaim, total) nodes := make([]*v1.Node, total) diff --git a/pkg/test/nodes.go b/pkg/test/nodes.go index 055bab0845..b925b08206 100644 --- a/pkg/test/nodes.go +++ b/pkg/test/nodes.go @@ -21,7 +21,6 @@ import ( v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "sigs.k8s.io/karpenter/pkg/apis/v1alpha5" "sigs.k8s.io/karpenter/pkg/apis/v1beta1" ) @@ -81,19 +80,3 @@ func NodeClaimLinkedNode(nodeClaim *v1beta1.NodeClaim) *v1.Node { }, ) } - -func MachineLinkedNode(machine *v1alpha5.Machine) *v1.Node { - return Node( - NodeOptions{ - ObjectMeta: metav1.ObjectMeta{ - Labels: machine.Labels, - Annotations: machine.Annotations, - Finalizers: machine.Finalizers, - }, - Taints: append(machine.Spec.Taints, machine.Spec.StartupTaints...), - Capacity: machine.Status.Capacity, - Allocatable: machine.Status.Allocatable, - ProviderID: machine.Status.ProviderID, - }, - ) -} diff --git a/pkg/test/provisioner.go b/pkg/test/provisioner.go deleted file mode 100644 index 3f2958b869..0000000000 --- a/pkg/test/provisioner.go +++ /dev/null @@ -1,99 +0,0 @@ -/* -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package test - -import ( - "encoding/json" - "fmt" - - "github.com/imdario/mergo" - "github.com/samber/lo" - v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - - "sigs.k8s.io/karpenter/pkg/apis/v1alpha5" -) - -// ProvisionerOptions customizes a Provisioner. -type ProvisionerOptions struct { - metav1.ObjectMeta - Limits v1.ResourceList - Provider interface{} - ProviderRef *v1alpha5.MachineTemplateRef - Kubelet *v1alpha5.KubeletConfiguration - Annotations map[string]string - Labels map[string]string - Taints []v1.Taint - StartupTaints []v1.Taint - Requirements []v1.NodeSelectorRequirement - Status v1alpha5.ProvisionerStatus - TTLSecondsUntilExpired *int64 - Weight *int32 - TTLSecondsAfterEmpty *int64 - Consolidation *v1alpha5.Consolidation -} - -// Provisioner creates a test provisioner with defaults that can be overridden by ProvisionerOptions. -// Overrides are applied in order, with a last write wins semantic. -func Provisioner(overrides ...ProvisionerOptions) *v1alpha5.Provisioner { - options := ProvisionerOptions{} - for _, opts := range overrides { - if err := mergo.Merge(&options, opts, mergo.WithOverride); err != nil { - panic(fmt.Sprintf("Failed to merge provisioner options: %s", err)) - } - } - if options.Name == "" { - options.Name = RandomName() - } - if options.Limits == nil { - options.Limits = v1.ResourceList{v1.ResourceCPU: resource.MustParse("2000")} - } - raw := &runtime.RawExtension{} - lo.Must0(raw.UnmarshalJSON(lo.Must(json.Marshal(options.Provider)))) - - provisioner := &v1alpha5.Provisioner{ - ObjectMeta: ObjectMeta(options.ObjectMeta), - Spec: v1alpha5.ProvisionerSpec{ - Requirements: options.Requirements, - KubeletConfiguration: options.Kubelet, - ProviderRef: options.ProviderRef, - Taints: options.Taints, - StartupTaints: options.StartupTaints, - Annotations: options.Annotations, - Labels: lo.Assign(options.Labels, map[string]string{DiscoveryLabel: "unspecified"}), // For node cleanup discovery - Limits: &v1alpha5.Limits{Resources: options.Limits}, - TTLSecondsAfterEmpty: options.TTLSecondsAfterEmpty, - TTLSecondsUntilExpired: options.TTLSecondsUntilExpired, - Weight: options.Weight, - Consolidation: options.Consolidation, - Provider: raw, - }, - Status: options.Status, - } - - if options.ProviderRef == nil { - if options.Provider == nil { - options.Provider = struct{}{} - } - provider, err := json.Marshal(options.Provider) - if err != nil { - panic(err.Error()) - } - provisioner.Spec.Provider = &runtime.RawExtension{Raw: provider} - } - return provisioner -} diff --git a/pkg/utils/nodeclaim/nodeclaim.go b/pkg/utils/nodeclaim/nodeclaim.go index 64aefffe17..ff0a9bc898 100644 --- a/pkg/utils/nodeclaim/nodeclaim.go +++ b/pkg/utils/nodeclaim/nodeclaim.go @@ -281,10 +281,6 @@ func DriftedCounter(nodeClaim *v1beta1.NodeClaim, driftType string) prometheus.C } func UpdateNodeOwnerReferences(nodeClaim *v1beta1.NodeClaim, node *v1.Node) *v1.Node { - // Remove any provisioner owner references since we own them - node.OwnerReferences = lo.Reject(node.OwnerReferences, func(o metav1.OwnerReference, _ int) bool { - return o.Kind == "Provisioner" - }) node.OwnerReferences = append(node.OwnerReferences, metav1.OwnerReference{ APIVersion: v1beta1.SchemeGroupVersion.String(), Kind: "NodeClaim", diff --git a/pkg/utils/nodeclaim/suite_test.go b/pkg/utils/nodeclaim/suite_test.go index 10464b68fc..0f929cdd07 100644 --- a/pkg/utils/nodeclaim/suite_test.go +++ b/pkg/utils/nodeclaim/suite_test.go @@ -33,7 +33,6 @@ import ( "sigs.k8s.io/karpenter/pkg/operator/scheme" . "sigs.k8s.io/karpenter/pkg/test/expectations" - "sigs.k8s.io/karpenter/pkg/apis/v1alpha5" "sigs.k8s.io/karpenter/pkg/apis/v1beta1" "sigs.k8s.io/karpenter/pkg/scheduling" "sigs.k8s.io/karpenter/pkg/test" @@ -67,16 +66,16 @@ var _ = Describe("NodeClaimUtils", func() { node = test.Node(test.NodeOptions{ ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{ - v1.LabelTopologyZone: "test-zone-1", - v1.LabelTopologyRegion: "test-region", - "test-label-key": "test-label-value", - "test-label-key2": "test-label-value2", - v1alpha5.LabelNodeRegistered: "true", - v1alpha5.LabelNodeInitialized: "true", - v1alpha5.ProvisionerNameLabelKey: "default", - v1alpha5.LabelCapacityType: v1alpha5.CapacityTypeOnDemand, - v1.LabelOSStable: "linux", - v1.LabelInstanceTypeStable: "test-instance-type", + v1.LabelTopologyZone: "test-zone-1", + v1.LabelTopologyRegion: "test-region", + "test-label-key": "test-label-value", + "test-label-key2": "test-label-value2", + v1beta1.NodeRegisteredLabelKey: "true", + v1beta1.NodeInitializedLabelKey: "true", + v1beta1.NodePoolLabelKey: "default", + v1beta1.CapacityTypeLabelKey: v1beta1.CapacityTypeOnDemand, + v1.LabelOSStable: "linux", + v1.LabelInstanceTypeStable: "test-instance-type", }, Annotations: map[string]string{ "test-annotation-key": "test-annotation-value", diff --git a/pkg/utils/nodepool/nodepool.go b/pkg/utils/nodepool/nodepool.go index 05a80aa0d5..0480ad355c 100644 --- a/pkg/utils/nodepool/nodepool.go +++ b/pkg/utils/nodepool/nodepool.go @@ -16,87 +16,13 @@ package nodepool import ( "context" - "fmt" - "time" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" - "github.com/samber/lo" - - "sigs.k8s.io/karpenter/pkg/apis/v1alpha5" "sigs.k8s.io/karpenter/pkg/apis/v1beta1" ) -func New(provisioner *v1alpha5.Provisioner) *v1beta1.NodePool { - np := &v1beta1.NodePool{ - ObjectMeta: provisioner.ObjectMeta, - Spec: v1beta1.NodePoolSpec{ - Template: v1beta1.NodeClaimTemplate{ - ObjectMeta: v1beta1.ObjectMeta{ - Annotations: provisioner.Spec.Annotations, - Labels: provisioner.Spec.Labels, - }, - Spec: v1beta1.NodeClaimSpec{ - Taints: provisioner.Spec.Taints, - StartupTaints: provisioner.Spec.StartupTaints, - Requirements: provisioner.Spec.Requirements, - Kubelet: NewKubeletConfiguration(provisioner.Spec.KubeletConfiguration), - NodeClassRef: NewNodeClassReference(provisioner.Spec.ProviderRef), - }, - }, - Weight: provisioner.Spec.Weight, - }, - Status: v1beta1.NodePoolStatus{ - Resources: provisioner.Status.Resources, - }, - } - if provisioner.Spec.TTLSecondsUntilExpired != nil { - np.Spec.Disruption.ExpireAfter.Duration = lo.ToPtr(lo.Must(time.ParseDuration(fmt.Sprintf("%ds", lo.FromPtr[int64](provisioner.Spec.TTLSecondsUntilExpired))))) - } - if provisioner.Spec.Consolidation != nil && lo.FromPtr(provisioner.Spec.Consolidation.Enabled) { - np.Spec.Disruption.ConsolidationPolicy = v1beta1.ConsolidationPolicyWhenUnderutilized - } else if provisioner.Spec.TTLSecondsAfterEmpty != nil { - np.Spec.Disruption.ConsolidationPolicy = v1beta1.ConsolidationPolicyWhenEmpty - np.Spec.Disruption.ConsolidateAfter = &v1beta1.NillableDuration{Duration: lo.ToPtr(lo.Must(time.ParseDuration(fmt.Sprintf("%ds", lo.FromPtr[int64](provisioner.Spec.TTLSecondsAfterEmpty)))))} - } - if provisioner.Spec.Limits != nil { - np.Spec.Limits = v1beta1.Limits(provisioner.Spec.Limits.Resources) - } - return np -} - -func NewKubeletConfiguration(kc *v1alpha5.KubeletConfiguration) *v1beta1.KubeletConfiguration { - if kc == nil { - return nil - } - return &v1beta1.KubeletConfiguration{ - ClusterDNS: kc.ClusterDNS, - MaxPods: kc.MaxPods, - PodsPerCore: kc.PodsPerCore, - SystemReserved: kc.SystemReserved, - KubeReserved: kc.KubeReserved, - EvictionHard: kc.EvictionHard, - EvictionSoft: kc.EvictionSoft, - EvictionSoftGracePeriod: kc.EvictionSoftGracePeriod, - EvictionMaxPodGracePeriod: kc.EvictionMaxPodGracePeriod, - ImageGCHighThresholdPercent: kc.ImageGCHighThresholdPercent, - ImageGCLowThresholdPercent: kc.ImageGCLowThresholdPercent, - CPUCFSQuota: kc.CPUCFSQuota, - } -} - -func NewNodeClassReference(pr *v1alpha5.MachineTemplateRef) *v1beta1.NodeClassReference { - if pr == nil { - return nil - } - return &v1beta1.NodeClassReference{ - Kind: pr.Kind, - Name: pr.Name, - APIVersion: pr.APIVersion, - } -} - func Get(ctx context.Context, c client.Client, name string) (*v1beta1.NodePool, error) { nodePool := &v1beta1.NodePool{} if err := c.Get(ctx, types.NamespacedName{Name: name}, nodePool); err != nil { diff --git a/pkg/utils/nodepool/suite_test.go b/pkg/utils/nodepool/suite_test.go index 5f0df194a7..3cb96c4f64 100644 --- a/pkg/utils/nodepool/suite_test.go +++ b/pkg/utils/nodepool/suite_test.go @@ -17,22 +17,16 @@ package nodepool_test import ( "context" "testing" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "github.com/samber/lo" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "knative.dev/pkg/apis" . "knative.dev/pkg/logging/testing" - "knative.dev/pkg/ptr" "sigs.k8s.io/controller-runtime/pkg/client" karpenterapis "sigs.k8s.io/karpenter/pkg/apis" - "sigs.k8s.io/karpenter/pkg/apis/v1alpha5" "sigs.k8s.io/karpenter/pkg/apis/v1beta1" "sigs.k8s.io/karpenter/pkg/operator/scheme" "sigs.k8s.io/karpenter/pkg/test" @@ -62,172 +56,6 @@ var _ = AfterEach(func() { }) var _ = Describe("NodePoolUtils", func() { - var provisioner *v1alpha5.Provisioner - BeforeEach(func() { - provisioner = test.Provisioner(test.ProvisionerOptions{ - Limits: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("10"), - v1.ResourceMemory: resource.MustParse("10Mi"), - v1.ResourceEphemeralStorage: resource.MustParse("1000Gi"), - }, - ProviderRef: &v1alpha5.MachineTemplateRef{ - Kind: "MachineTemplate", - APIVersion: "test.cloudprovider/v1", - Name: "default", - }, - Kubelet: &v1alpha5.KubeletConfiguration{ - ContainerRuntime: ptr.String("containerd"), - MaxPods: ptr.Int32(110), - PodsPerCore: ptr.Int32(10), - SystemReserved: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("200m"), - v1.ResourceMemory: resource.MustParse("200Mi"), - v1.ResourceEphemeralStorage: resource.MustParse("1Gi"), - }, - KubeReserved: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("200m"), - v1.ResourceMemory: resource.MustParse("200Mi"), - v1.ResourceEphemeralStorage: resource.MustParse("1Gi"), - }, - EvictionHard: map[string]string{ - "memory.available": "5%", - "nodefs.available": "5%", - "nodefs.inodesFree": "5%", - "imagefs.available": "5%", - "imagefs.inodesFree": "5%", - "pid.available": "3%", - }, - EvictionSoft: map[string]string{ - "memory.available": "10%", - "nodefs.available": "10%", - "nodefs.inodesFree": "10%", - "imagefs.available": "10%", - "imagefs.inodesFree": "10%", - "pid.available": "6%", - }, - EvictionSoftGracePeriod: map[string]metav1.Duration{ - "memory.available": {Duration: time.Minute * 2}, - "nodefs.available": {Duration: time.Minute * 2}, - "nodefs.inodesFree": {Duration: time.Minute * 2}, - "imagefs.available": {Duration: time.Minute * 2}, - "imagefs.inodesFree": {Duration: time.Minute * 2}, - "pid.available": {Duration: time.Minute * 2}, - }, - EvictionMaxPodGracePeriod: ptr.Int32(120), - ImageGCHighThresholdPercent: ptr.Int32(50), - ImageGCLowThresholdPercent: ptr.Int32(10), - CPUCFSQuota: ptr.Bool(false), - }, - Annotations: map[string]string{ - "test-annotation-key": "test-annotation-value", - "test-annotation-key2": "test-annotation-value2", - }, - Labels: map[string]string{ - "test-label-key": "test-label-value", - "test-label-key2": "test-label-value2", - }, - Taints: []v1.Taint{ - { - Key: "test-taint-key", - Effect: v1.TaintEffectNoSchedule, - Value: "test-taint-value", - }, - { - Key: "test-taint-key2", - Effect: v1.TaintEffectNoExecute, - Value: "test-taint-value2", - }, - }, - StartupTaints: []v1.Taint{ - { - Key: "test-startup-taint-key", - Effect: v1.TaintEffectNoSchedule, - Value: "test-startup-taint-value", - }, - { - Key: "test-startup-taint-key2", - Effect: v1.TaintEffectNoExecute, - Value: "test-startup-taint-value2", - }, - }, - Requirements: []v1.NodeSelectorRequirement{ - { - Key: v1.LabelTopologyZone, - Operator: v1.NodeSelectorOpIn, - Values: []string{"test-zone-1", "test-zone-2"}, - }, - { - Key: v1alpha5.LabelCapacityType, - Operator: v1.NodeSelectorOpIn, - Values: []string{v1alpha5.CapacityTypeOnDemand}, - }, - { - Key: v1.LabelHostname, - Operator: v1.NodeSelectorOpExists, - }, - }, - TTLSecondsUntilExpired: lo.ToPtr[int64](2592000), - TTLSecondsAfterEmpty: lo.ToPtr[int64](30), - Consolidation: &v1alpha5.Consolidation{ - Enabled: lo.ToPtr(true), - }, - Weight: lo.ToPtr[int32](100), - Status: v1alpha5.ProvisionerStatus{ - Resources: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("5"), - v1.ResourceMemory: resource.MustParse("5Mi"), - v1.ResourceEphemeralStorage: resource.MustParse("500Gi"), - }, - LastScaleTime: &apis.VolatileTime{Inner: metav1.Time{Time: time.Now()}}, - }, - }) - provisioner.Spec.Provider = nil - }) - It("should convert a Provisioner to a NodePool", func() { - nodePool := nodepoolutil.New(provisioner) - for k, v := range provisioner.Annotations { - Expect(nodePool.Annotations).To(HaveKeyWithValue(k, v)) - } - for k, v := range provisioner.Labels { - Expect(nodePool.Labels).To(HaveKeyWithValue(k, v)) - } - for k, v := range provisioner.Spec.Annotations { - Expect(nodePool.Spec.Template.Annotations).To(HaveKeyWithValue(k, v)) - } - for k, v := range provisioner.Spec.Labels { - Expect(nodePool.Spec.Template.Labels).To(HaveKeyWithValue(k, v)) - } - - Expect(nodePool.Spec.Template.Spec.Taints).To(Equal(provisioner.Spec.Taints)) - Expect(nodePool.Spec.Template.Spec.StartupTaints).To(Equal(provisioner.Spec.StartupTaints)) - Expect(nodePool.Spec.Template.Spec.Requirements).To(Equal(provisioner.Spec.Requirements)) - - Expect(nodePool.Spec.Template.Spec.Kubelet.ClusterDNS).To(Equal(provisioner.Spec.KubeletConfiguration.ClusterDNS)) - Expect(nodePool.Spec.Template.Spec.Kubelet.MaxPods).To(Equal(provisioner.Spec.KubeletConfiguration.MaxPods)) - Expect(nodePool.Spec.Template.Spec.Kubelet.PodsPerCore).To(Equal(provisioner.Spec.KubeletConfiguration.PodsPerCore)) - Expect(nodePool.Spec.Template.Spec.Kubelet.SystemReserved).To(Equal(provisioner.Spec.KubeletConfiguration.SystemReserved)) - Expect(nodePool.Spec.Template.Spec.Kubelet.KubeReserved).To(Equal(provisioner.Spec.KubeletConfiguration.KubeReserved)) - Expect(nodePool.Spec.Template.Spec.Kubelet.EvictionHard).To(Equal(provisioner.Spec.KubeletConfiguration.EvictionHard)) - Expect(nodePool.Spec.Template.Spec.Kubelet.EvictionSoft).To(Equal(provisioner.Spec.KubeletConfiguration.EvictionSoft)) - Expect(nodePool.Spec.Template.Spec.Kubelet.EvictionSoftGracePeriod).To(Equal(provisioner.Spec.KubeletConfiguration.EvictionSoftGracePeriod)) - Expect(nodePool.Spec.Template.Spec.Kubelet.EvictionMaxPodGracePeriod).To(Equal(provisioner.Spec.KubeletConfiguration.EvictionMaxPodGracePeriod)) - Expect(nodePool.Spec.Template.Spec.Kubelet.ImageGCHighThresholdPercent).To(Equal(provisioner.Spec.KubeletConfiguration.ImageGCHighThresholdPercent)) - Expect(nodePool.Spec.Template.Spec.Kubelet.ImageGCLowThresholdPercent).To(Equal(provisioner.Spec.KubeletConfiguration.ImageGCLowThresholdPercent)) - Expect(nodePool.Spec.Template.Spec.Kubelet.CPUCFSQuota).To(Equal(provisioner.Spec.KubeletConfiguration.CPUCFSQuota)) - - Expect(nodePool.Spec.Template.Spec.NodeClassRef.Kind).To(Equal(provisioner.Spec.ProviderRef.Kind)) - Expect(nodePool.Spec.Template.Spec.NodeClassRef.APIVersion).To(Equal(provisioner.Spec.ProviderRef.APIVersion)) - Expect(nodePool.Spec.Template.Spec.NodeClassRef.Name).To(Equal(provisioner.Spec.ProviderRef.Name)) - - Expect(nodePool.Spec.Disruption.ConsolidationPolicy).To(Equal(v1beta1.ConsolidationPolicyWhenUnderutilized)) - Expect(nodePool.Spec.Disruption.ExpireAfter.Duration.Seconds()).To(BeNumerically("==", lo.FromPtr(provisioner.Spec.TTLSecondsUntilExpired))) - Expect(nodePool.Spec.Disruption.ConsolidateAfter).To(BeNil()) - - ExpectResources(v1.ResourceList(nodePool.Spec.Limits), provisioner.Spec.Limits.Resources) - Expect(lo.FromPtr(nodePool.Spec.Weight)).To(BeNumerically("==", lo.FromPtr(provisioner.Spec.Weight))) - - ExpectResources(nodePool.Status.Resources, provisioner.Status.Resources) - }) It("should patch the status on a NodePool", func() { nodePool := test.NodePool(v1beta1.NodePool{ Spec: v1beta1.NodePoolSpec{ diff --git a/pkg/webhooks/webhooks.go b/pkg/webhooks/webhooks.go index dc66022bf9..013222f002 100644 --- a/pkg/webhooks/webhooks.go +++ b/pkg/webhooks/webhooks.go @@ -40,7 +40,6 @@ import ( "knative.dev/pkg/webhook/resourcesemantics/validation" "sigs.k8s.io/controller-runtime/pkg/healthz" - "sigs.k8s.io/karpenter/pkg/apis/v1alpha5" "sigs.k8s.io/karpenter/pkg/apis/v1beta1" "sigs.k8s.io/karpenter/pkg/operator/logging" "sigs.k8s.io/karpenter/pkg/operator/options" @@ -49,9 +48,8 @@ import ( const component = "webhook" var Resources = map[schema.GroupVersionKind]resourcesemantics.GenericCRD{ - v1alpha5.SchemeGroupVersion.WithKind("Provisioner"): &v1alpha5.Provisioner{}, - v1beta1.SchemeGroupVersion.WithKind("NodePool"): &v1beta1.NodePool{}, - v1beta1.SchemeGroupVersion.WithKind("NodeClaim"): &v1beta1.NodeClaim{}, + v1beta1.SchemeGroupVersion.WithKind("NodePool"): &v1beta1.NodePool{}, + v1beta1.SchemeGroupVersion.WithKind("NodeClaim"): &v1beta1.NodeClaim{}, } func NewWebhooks() []knativeinjection.ControllerConstructor { From e38fb2fe3a91adf69ecd60c2c07b29326d60a543 Mon Sep 17 00:00:00 2001 From: njtran Date: Mon, 4 Dec 2023 20:06:24 -0800 Subject: [PATCH 2/4] comments --- pkg/controllers/metrics/node/controller.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/controllers/metrics/node/controller.go b/pkg/controllers/metrics/node/controller.go index dabccb9a7b..f1886000a3 100644 --- a/pkg/controllers/metrics/node/controller.go +++ b/pkg/controllers/metrics/node/controller.go @@ -101,6 +101,8 @@ var ( func nodeLabelNames() []string { return append( + // WellKnownLabels includes the nodepool label, so we don't need to add it as its own item here. + // if we do, Prometheus will panic since there would be duplicate labels. sets.New(lo.Values(wellKnownLabels)...).UnsortedList(), resourceType, nodeName, @@ -173,6 +175,7 @@ func getNodeLabels(node *v1.Node, resourceTypeName string) prometheus.Labels { metricLabels := prometheus.Labels{} metricLabels[resourceType] = resourceTypeName metricLabels[nodeName] = node.Name + metricLabels[v1beta1.NodePoolLabelKey] = node.Labels[v1beta1.NodePoolLabelKey] metricLabels[nodePhase] = string(node.Status.Phase) // Populate well known labels From 81a79e8098f92364a36d4adfdba4719cf46e9b2e Mon Sep 17 00:00:00 2001 From: njtran Date: Mon, 4 Dec 2023 20:19:33 -0800 Subject: [PATCH 3/4] fix test and comments --- pkg/controllers/metrics/node/controller.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pkg/controllers/metrics/node/controller.go b/pkg/controllers/metrics/node/controller.go index f1886000a3..11597ee3f0 100644 --- a/pkg/controllers/metrics/node/controller.go +++ b/pkg/controllers/metrics/node/controller.go @@ -102,7 +102,7 @@ var ( func nodeLabelNames() []string { return append( // WellKnownLabels includes the nodepool label, so we don't need to add it as its own item here. - // if we do, Prometheus will panic since there would be duplicate labels. + // If we do, prometheus will panic since there would be duplicate labels. sets.New(lo.Values(wellKnownLabels)...).UnsortedList(), resourceType, nodeName, @@ -175,7 +175,6 @@ func getNodeLabels(node *v1.Node, resourceTypeName string) prometheus.Labels { metricLabels := prometheus.Labels{} metricLabels[resourceType] = resourceTypeName metricLabels[nodeName] = node.Name - metricLabels[v1beta1.NodePoolLabelKey] = node.Labels[v1beta1.NodePoolLabelKey] metricLabels[nodePhase] = string(node.Status.Phase) // Populate well known labels From 159eb964d5edf4c63655a79ef65b1a8908ba14ea Mon Sep 17 00:00:00 2001 From: njtran Date: Mon, 4 Dec 2023 20:28:37 -0800 Subject: [PATCH 4/4] remove unneeded comment --- pkg/controllers/metrics/node/controller.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/controllers/metrics/node/controller.go b/pkg/controllers/metrics/node/controller.go index 11597ee3f0..e0d7cd7783 100644 --- a/pkg/controllers/metrics/node/controller.go +++ b/pkg/controllers/metrics/node/controller.go @@ -186,7 +186,6 @@ func getNodeLabels(node *v1.Node, resourceTypeName string) prometheus.Labels { func getWellKnownLabels() map[string]string { labels := make(map[string]string) - // TODO @joinnis: Remove v1alpha5 well-known labels in favor of only v1beta1 well-known labels after v1alpha5 is dropped for wellKnownLabel := range v1beta1.WellKnownLabels { if parts := strings.Split(wellKnownLabel, "/"); len(parts) == 2 { label := parts[1]