/
instancetypes.go
256 lines (235 loc) · 9.85 KB
/
instancetypes.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
/*
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package aws
import (
"context"
"fmt"
"net/http"
"sync"
"time"
"github.com/aws/karpenter/pkg/apis/provisioning/v1alpha5"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/ec2"
"github.com/aws/aws-sdk-go/service/ec2/ec2iface"
"github.com/mitchellh/hashstructure/v2"
"github.com/patrickmn/go-cache"
"github.com/samber/lo"
"k8s.io/apimachinery/pkg/util/sets"
"knative.dev/pkg/logging"
"github.com/aws/karpenter/pkg/cloudprovider"
"github.com/aws/karpenter/pkg/cloudprovider/aws/apis/v1alpha1"
"github.com/aws/karpenter/pkg/utils/functional"
"github.com/aws/karpenter/pkg/utils/injection"
"github.com/aws/karpenter/pkg/utils/pretty"
)
const (
InstanceTypesCacheKey = "types"
InstanceTypeZonesCacheKeyPrefix = "zones:"
InstanceTypesAndZonesCacheTTL = 5 * time.Minute
UnfulfillableCapacityErrorCacheTTL = 3 * time.Minute
)
type InstanceTypeProvider struct {
sync.Mutex
region string
ec2api ec2iface.EC2API
subnetProvider *SubnetProvider
pricingProvider *PricingProvider
// Has one cache entry for all the instance types (key: InstanceTypesCacheKey)
// Has one cache entry for all the zones for each subnet selector (key: InstanceTypesZonesCacheKeyPrefix:<hash_of_selector>)
// Values cached *before* considering insufficient capacity errors from the unavailableOfferings cache.
cache *cache.Cache
// key: <capacityType>:<instanceType>:<zone>, value: struct{}{}
unavailableOfferings *cache.Cache
cm *pretty.ChangeMonitor
}
func NewInstanceTypeProvider(ctx context.Context, sess *session.Session, options cloudprovider.Options, ec2api ec2iface.EC2API, subnetProvider *SubnetProvider) *InstanceTypeProvider {
return &InstanceTypeProvider{
ec2api: ec2api,
region: *sess.Config.Region,
subnetProvider: subnetProvider,
pricingProvider: NewPricingProvider(ctx,
NewPricingAPI(sess, *sess.Config.Region),
ec2api,
*sess.Config.Region,
injection.GetOptions(ctx).AWSIsolatedVPC, options.StartAsync),
cache: cache.New(InstanceTypesAndZonesCacheTTL, CacheCleanupInterval),
unavailableOfferings: cache.New(UnfulfillableCapacityErrorCacheTTL, CacheCleanupInterval),
cm: pretty.NewChangeMonitor(),
}
}
// Get all instance type options
func (p *InstanceTypeProvider) Get(ctx context.Context, provider *v1alpha1.AWS, kc *v1alpha5.KubeletConfiguration) ([]cloudprovider.InstanceType, error) {
p.Lock()
defer p.Unlock()
// Get InstanceTypes from EC2
instanceTypes, err := p.getInstanceTypes(ctx)
if err != nil {
return nil, err
}
// Get Viable EC2 Purchase offerings
instanceTypeZones, err := p.getInstanceTypeZones(ctx, provider)
if err != nil {
return nil, err
}
var result []cloudprovider.InstanceType
for _, i := range instanceTypes {
instanceTypeName := aws.StringValue(i.InstanceType)
instanceType := NewInstanceType(ctx, i, kc, p.region, provider, p.createOfferings(ctx, i, instanceTypeZones[instanceTypeName]))
result = append(result, instanceType)
}
return result, nil
}
func (p *InstanceTypeProvider) LivenessProbe(req *http.Request) error {
p.Lock()
//nolint: staticcheck
p.Unlock()
if err := p.subnetProvider.LivenessProbe(req); err != nil {
return err
}
if err := p.pricingProvider.LivenessProbe(req); err != nil {
return err
}
return nil
}
func (p *InstanceTypeProvider) createOfferings(ctx context.Context, instanceType *ec2.InstanceTypeInfo, zones sets.String) []cloudprovider.Offering {
offerings := []cloudprovider.Offering{}
for zone := range zones {
// while usage classes should be a distinct set, there's no guarantee of that
for capacityType := range sets.NewString(aws.StringValueSlice(instanceType.SupportedUsageClasses)...) {
// exclude any offerings that have recently seen an insufficient capacity error from EC2
_, isUnavailable := p.unavailableOfferings.Get(UnavailableOfferingsCacheKey(*instanceType.InstanceType, zone, capacityType))
var price float64
var ok bool
switch capacityType {
case ec2.UsageClassTypeSpot:
price, ok = p.pricingProvider.SpotPrice(*instanceType.InstanceType, zone)
case ec2.UsageClassTypeOnDemand:
price, ok = p.pricingProvider.OnDemandPrice(*instanceType.InstanceType)
default:
logging.FromContext(ctx).Errorf("Received unknown capacity type %s for instance type %s", capacityType, *instanceType.InstanceType)
continue
}
available := !isUnavailable && ok
offerings = append(offerings, cloudprovider.Offering{
Zone: zone,
CapacityType: capacityType,
Price: price,
Available: available,
})
}
}
return offerings
}
func (p *InstanceTypeProvider) getInstanceTypeZones(ctx context.Context, provider *v1alpha1.AWS) (map[string]sets.String, error) {
subnetSelectorHash, err := hashstructure.Hash(provider.SubnetSelector, hashstructure.FormatV2, nil)
if err != nil {
return nil, fmt.Errorf("failed to hash the subnet selector: %w", err)
}
cacheKey := fmt.Sprintf("%s%016x", InstanceTypeZonesCacheKeyPrefix, subnetSelectorHash)
if cached, ok := p.cache.Get(cacheKey); ok {
return cached.(map[string]sets.String), nil
}
// Constrain AZs from subnets
subnets, err := p.subnetProvider.Get(ctx, provider)
if err != nil {
return nil, err
}
zones := sets.NewString(lo.Map(subnets, func(subnet *ec2.Subnet, _ int) string {
return aws.StringValue(subnet.AvailabilityZone)
})...)
// Get offerings from EC2
instanceTypeZones := map[string]sets.String{}
if err := p.ec2api.DescribeInstanceTypeOfferingsPagesWithContext(ctx, &ec2.DescribeInstanceTypeOfferingsInput{LocationType: aws.String("availability-zone")},
func(output *ec2.DescribeInstanceTypeOfferingsOutput, lastPage bool) bool {
for _, offering := range output.InstanceTypeOfferings {
if zones.Has(aws.StringValue(offering.Location)) {
if _, ok := instanceTypeZones[aws.StringValue(offering.InstanceType)]; !ok {
instanceTypeZones[aws.StringValue(offering.InstanceType)] = sets.NewString()
}
instanceTypeZones[aws.StringValue(offering.InstanceType)].Insert(aws.StringValue(offering.Location))
}
}
return true
}); err != nil {
return nil, fmt.Errorf("describing instance type zone offerings, %w", err)
}
if p.cm.HasChanged("zonal-offerings", provider.SubnetSelector) {
logging.FromContext(ctx).Debugf("Discovered EC2 instance types zonal offerings for subnets %s", pretty.Concise(provider.SubnetSelector))
}
p.cache.SetDefault(cacheKey, instanceTypeZones)
return instanceTypeZones, nil
}
// getInstanceTypes retrieves all instance types from the ec2 DescribeInstanceTypes API using some opinionated filters
func (p *InstanceTypeProvider) getInstanceTypes(ctx context.Context) (map[string]*ec2.InstanceTypeInfo, error) {
if cached, ok := p.cache.Get(InstanceTypesCacheKey); ok {
return cached.(map[string]*ec2.InstanceTypeInfo), nil
}
instanceTypes := map[string]*ec2.InstanceTypeInfo{}
if err := p.ec2api.DescribeInstanceTypesPagesWithContext(ctx, &ec2.DescribeInstanceTypesInput{
Filters: []*ec2.Filter{
{
Name: aws.String("supported-virtualization-type"),
Values: []*string{aws.String("hvm")},
},
{
Name: aws.String("processor-info.supported-architecture"),
Values: aws.StringSlice([]string{"x86_64", "arm64"}),
},
},
}, func(page *ec2.DescribeInstanceTypesOutput, lastPage bool) bool {
for _, instanceType := range page.InstanceTypes {
if p.filter(instanceType) {
instanceTypes[aws.StringValue(instanceType.InstanceType)] = instanceType
}
}
return true
}); err != nil {
return nil, fmt.Errorf("fetching instance types using ec2.DescribeInstanceTypes, %w", err)
}
if p.cm.HasChanged("instance-types", instanceTypes) {
logging.FromContext(ctx).Debugf("Discovered %d EC2 instance types", len(instanceTypes))
}
p.cache.SetDefault(InstanceTypesCacheKey, instanceTypes)
return instanceTypes, nil
}
// filter the instance types to include useful ones for Kubernetes
func (p *InstanceTypeProvider) filter(instanceType *ec2.InstanceTypeInfo) bool {
if instanceType.FpgaInfo != nil {
return false
}
if functional.HasAnyPrefix(aws.StringValue(instanceType.InstanceType),
// G2 instances have an older GPU not supported by the nvidia plugin. This causes the allocatable # of gpus
// to be set to zero on startup as the plugin considers the GPU unhealthy.
"g2",
) {
return false
}
return true
}
// CacheUnavailable allows the InstanceProvider to communicate recently observed temporary capacity shortages in
// the provided offerings
func (p *InstanceTypeProvider) CacheUnavailable(ctx context.Context, fleetErr *ec2.CreateFleetError, capacityType string) {
instanceType := aws.StringValue(fleetErr.LaunchTemplateAndOverrides.Overrides.InstanceType)
zone := aws.StringValue(fleetErr.LaunchTemplateAndOverrides.Overrides.AvailabilityZone)
logging.FromContext(ctx).Debugf("%s for offering { instanceType: %s, zone: %s, capacityType: %s }, avoiding for %s",
aws.StringValue(fleetErr.ErrorCode),
instanceType,
zone,
capacityType,
UnfulfillableCapacityErrorCacheTTL)
// even if the key is already in the cache, we still need to call Set to extend the cached entry's TTL
p.unavailableOfferings.SetDefault(UnavailableOfferingsCacheKey(instanceType, zone, capacityType), struct{}{})
}
func UnavailableOfferingsCacheKey(instanceType string, zone string, capacityType string) string {
return fmt.Sprintf("%s:%s:%s", capacityType, instanceType, zone)
}