Rebase 1.17.0

OWNERS

@@ -6,6 +6,7 @@ approvers:
   - ingvagabund
   - bison
 reviewers:
-  - paulfantom
-  - spangenberg
-  - vikaschoudhary16
+  - enxebre
+  - frobware
+  - ingvagabund
+  - bison

cluster-autoscaler/FAQ.md

@@ -657,7 +657,7 @@ The following startup parameters are supported for cluster autoscaler:
 | `ok-total-unready-count` | Number of allowed unready nodes, irrespective of max-total-unready-percentage  | 3
 | `max-node-provision-time` | Maximum time CA waits for node to be provisioned | 15 minutes
 | `nodes` | sets min,max size and other configuration data for a node group in a format accepted by cloud provider. Can be used multiple times. Format: <min>:<max>:<other...> | ""
-| `node-group-auto-discovery` | One or more definition(s) of node group auto-discovery.<br>A definition is expressed `<name of discoverer>:[<key>[=<value>]]`<br>The `aws` and `gce` cloud providers are currently supported. AWS matches by ASG tags, e.g. `asg:tag=tagKey,anotherTagKey`<br>GCE matches by IG name prefix, and requires you to specify min and max nodes per IG, e.g. `mig:namePrefix=pfx,min=0,max=10`<br>Can be used multiple times | ""
+| `node-group-auto-discovery` | One or more definition(s) of node group auto-discovery.<br>A definition is expressed `<name of discoverer>:[<key>[=<value>]]`<br>The `aws`, `gce`, and `azure` cloud providers are currently supported. AWS matches by ASG tags, e.g. `asg:tag=tagKey,anotherTagKey`<br>GCE matches by IG name prefix, and requires you to specify min and max nodes per IG, e.g. `mig:namePrefix=pfx,min=0,max=10`<br> Azure matches by tags on VMSS, e.g. `label:foo=bar`, and will auto-detect `min` and `max` tags on the VMSS to set scaling limits.<br>Can be used multiple times | ""
 | `estimator` | Type of resource estimator to be used in scale up | binpacking
 | `expander` | Type of node group expander to be used in scale up.  | random
 | `write-status-configmap` | Should CA write status information to a configmap  | true
@@ -674,6 +674,7 @@ The following startup parameters are supported for cluster autoscaler:
 | `leader-elect-renew-deadline` | The interval between attempts by the acting master to renew a leadership slot before it stops leading.<br>This must be less than or equal to the lease duration.<br>This is only applicable if leader election is enabled | 10 seconds
 | `leader-elect-retry-period` | The duration the clients should wait between attempting acquisition and renewal of a leadership.<br>This is only applicable if leader election is enabled | 2 seconds
 | `leader-elect-resource-lock` | The type of resource object that is used for locking during leader election.<br>Supported options are `endpoints` (default) and `configmaps` | "endpoints"
+| `aws-use-static-instance-list` | Should CA fetch instance types in runtime or use a static list. AWS only | false
 
 # Troubleshooting:
 

cluster-autoscaler/OWNERS

@@ -1,6 +1,12 @@
+# See the OWNERS docs: https://git.k8s.io/community/contributors/guide/owners.md
+
 approvers:
   - enxebre
   - frobware
+  - ingvagabund
   - bison
 reviewers:
-  - spangenberg
+  - enxebre
+  - frobware
+  - ingvagabund
+  - bison

cluster-autoscaler/README.md

@@ -25,6 +25,7 @@ Starting from Kubernetes 1.12, versioning scheme was changed to match Kubernetes
 
 | Kubernetes Version  | CA Version   |
 |--------|--------|
+| 1.16.X | 1.16.X  |
 | 1.15.X | 1.15.X  |
 | 1.14.X | 1.14.X  |
 | 1.13.X | 1.13.X  |

cluster-autoscaler/cloudprovider/aws/README.md

@@ -1,5 +1,5 @@
 # Cluster Autoscaler on AWS
-The cluster autoscaler on AWS scales worker nodes within any specified autoscaling group. It will run as a `Deployment` in your cluster. This README will go over some of the necessary steps required to get the cluster autoscaler up and running.
+The cluster autoscaler on AWS scales worker nodes within any specified autoscaling group. It will run as a `Deployment` in your cluster. This README covers the steps required to configure and run the cluster autoscaler.
 
 ## Kubernetes Version
 Cluster autoscaler must run on v1.3.0 or greater.
@@ -27,35 +27,23 @@ A minimum IAM policy would look like:
     ]
 }
 ```
+If you'd like to scale node groups from 0, an
+`autoscaling:DescribeLaunchConfigurations` or
+`ec2:DescribeLaunchTemplateVersions` permission is required depending on if you
+made your ASG with Launch Configuration or Launch Template.
 
-If you'd like to auto-discover node groups by specifying the `--node-group-auto-discovery` flag, a `DescribeTags` permission is also required:
+If you'd like the cluster autoscaler to [automatically
+discover](#auto-discovery-setup) EC2 AutoScalingGroups, the
+`autoscaling:DescribeTags` permission is also required.
 
-```json
-{
-    "Version": "2012-10-17",
-    "Statement": [
-        {
-            "Effect": "Allow",
-            "Action": [
-                "autoscaling:DescribeAutoScalingGroups",
-                "autoscaling:DescribeAutoScalingInstances",
-                "autoscaling:DescribeLaunchConfigurations",
-                "autoscaling:DescribeTags",
-                "autoscaling:SetDesiredCapacity",
-                "autoscaling:TerminateInstanceInAutoScalingGroup"
-            ],
-            "Resource": "*"
-        }
-    ]
-}
-```
-
-AWS supports ARNs for autoscaling groups. More information [here](https://docs.aws.amazon.com/autoscaling/latest/userguide/control-access-using-iam.html#policy-auto-scaling-resources).
+**NOTE**: You can restrict the target resources for the autoscaling actions by
+specifying autoscaling group ARNS. More information can be found
+[here](https://docs.aws.amazon.com/autoscaling/latest/userguide/control-access-using-iam.html#policy-auto-scaling-resources).
 
 ### Using AWS Credentials
-For on premise users like to scale out to AWS, above approach that attaching policy to nodegroup role won't work. Instead, you can create an aws secret manually and add following environment variables to cluster-autoscaler deployment manifest. Cluster autoscaler will use credential to authenticate and authorize itself. Please make sure your role has above permissions.
+For on premise users wishing to scale out to AWS, the above approach of attaching policy to a nodegroup role won't work. Instead, you can create an aws secret manually and add following environment variables to cluster-autoscaler deployment manifest. Cluster autoscaler will use credential to authenticate and authorize itself. Please make sure your role has above permissions.
 
-```
+```yaml
 apiVersion: v1
 kind: Secret
 metadata:
@@ -64,11 +52,10 @@ type: Opaque
 data:
   aws_access_key_id: BASE64_OF_YOUR_AWS_ACCESS_KEY_ID
   aws_secret_access_key: BASE64_OF_YOUR_AWS_SECRET_ACCESS_KEY
-
 ```
 Please check [guidance](https://kubernetes.io/docs/concepts/configuration/secret/#creating-a-secret-manually) for creating a secret manually.
 
-```
+```yaml
 env:
 - name: AWS_ACCESS_KEY_ID
   valueFrom:
@@ -82,7 +69,6 @@ env:
       key: aws_secret_access_key
 - name: AWS_REGION
   value: YOUR_AWS_REGION
-
 ```
 
 ## Deployment Specification
@@ -108,7 +94,6 @@ Please replace `{{ node_asg_min }}`, `{{ node_asg_max }}` and `{{ name }}` with
 kubectl apply -f examples/cluster-autoscaler-run-on-master.yaml
 ```
 
-
 ### Auto-Discovery Setup
 
 To run a cluster-autoscaler which auto-discovers ASGs with nodes use the `--node-group-auto-discovery` flag. For example, `--node-group-auto-discovery=asg:tag=k8s.io/cluster-autoscaler/enabled,k8s.io/cluster-autoscaler/<YOUR CLUSTER NAME>` will find the ASGs where those tag keys
@@ -210,11 +195,17 @@ spec:
 * Create an ASG with a MixedInstancesPolicy that refers to the newly-created LT.
 * Set LaunchTemplateOverrides to include the 'base' instance type r5.2xlarge and suitable alternatives, e.g. r5d.2xlarge, i3.2xlarge, r5a.2xlarge and r5ad.2xlarge. Differing processor types and speeds should be evaluated depending on your use-case(s).
 * Set [InstancesDistribution](https://docs.aws.amazon.com/autoscaling/ec2/APIReference/API_InstancesDistribution.html) according to your needs.
-* See [Allocation Strategies](https://docs.aws.amazon.com/autoscaling/ec2/userguide/asg-purchase-options.htlm#asg-allocation-strategies) for information about the ASG fulfils capacity from the specified instance types.
+* See [Allocation Strategies](https://docs.aws.amazon.com/autoscaling/ec2/userguide/asg-purchase-options.html#asg-allocation-strategies) for information about the ASG fulfils capacity from the specified instance types.
 * Repeat by creating other LTs and ASGs, for example c5.18xlarge and c5n.18xlarge or a bunch of similar burstable instances.
 
 See CloudFormation example [here](MixedInstancePolicy.md).
 
+## Use Static Instance List
+The set of the latest supported EC2 instance types will be fetched by the CA at run time. You can find all the available instance types in the CA logs.
+If your network access is restricted such that fetching this set is infeasible, you can specify the command-line flag `--aws-use-static-instance-list=true` to switch the CA back to its original use of a statically defined set.
+
+To refresh static list, please run `go run ec2_instance_types/gen.go` under `cluster-autoscaler/cloudprovider/aws/` and update `staticListLastUpdateTime` in `aws_util.go`
+
 ## Common Notes and Gotchas:
 - The `/etc/ssl/certs/ca-bundle.crt` should exist by default on ec2 instance in your EKS cluster. If you use other cluster privision tools like [kops](https://github.com/kubernetes/kops) with different operating systems other than Amazon Linux 2, please use `/etc/ssl/certs/ca-certificates.crt` or correct path on your host instead for the volume hostPath in your cluster autoscaler manifest.
 - Cluster autoscaler does not support Auto Scaling Groups which span multiple Availability Zones; instead you should use an Auto Scaling Group for each Availability Zone and enable the [--balance-similar-node-groups](../../FAQ.md#im-running-cluster-with-nodes-in-multiple-zones-for-ha-purposes-is-that-supported-by-cluster-autoscaler) feature. If you do use a single Auto Scaling Group that spans multiple Availability Zones you will find that AWS unexpectedly terminates nodes without them being drained because of the [rebalancing feature](https://docs.aws.amazon.com/autoscaling/ec2/userguide/auto-scaling-benefits.html#arch-AutoScalingMultiAZ).

cluster-autoscaler/cloudprovider/aws/auto_scaling_groups.go

@@ -19,11 +19,9 @@ package aws
 import (
 	"fmt"
 	"reflect"
-	"regexp"
 	"strings"
 	"sync"
 
-	"k8s.io/autoscaler/cluster-autoscaler/cloudprovider"
 	"k8s.io/autoscaler/cluster-autoscaler/config/dynamic"
 
 	"github.com/aws/aws-sdk-go/aws"
@@ -44,7 +42,7 @@ type asgCache struct {
 	service        autoScalingWrapper
 	interrupt      chan struct{}
 
-	asgAutoDiscoverySpecs []cloudprovider.ASGAutoDiscoveryConfig
+	asgAutoDiscoverySpecs []asgAutoDiscoveryConfig
 	explicitlyConfigured  map[AwsRef]bool
 }
 
@@ -72,7 +70,7 @@ type asg struct {
 	Tags                    []*autoscaling.TagDescription
 }
 
-func newASGCache(service autoScalingWrapper, explicitSpecs []string, autoDiscoverySpecs []cloudprovider.ASGAutoDiscoveryConfig) (*asgCache, error) {
+func newASGCache(service autoScalingWrapper, explicitSpecs []string, autoDiscoverySpecs []asgAutoDiscoveryConfig) (*asgCache, error) {
 	registry := &asgCache{
 		registeredAsgs:        make([]*asg, 0),
 		service:               service,
@@ -277,18 +275,17 @@ func (m *asgCache) DeleteInstances(instances []*AwsInstanceRef) error {
 				return err
 			}
 			klog.V(4).Infof(*resp.Activity.Description)
-		}
 
-		// Proactively decrement the size so autoscaler makes better decisions
-		commonAsg.curSize--
+			// Proactively decrement the size so autoscaler makes better decisions
+			commonAsg.curSize--
+		}
 	}
 	return nil
 }
 
 // isPlaceholderInstance checks if the given instance is only a placeholder
 func (m *asgCache) isPlaceholderInstance(instance *AwsInstanceRef) bool {
-	matched, _ := regexp.MatchString(fmt.Sprintf("^%s.*\\d+$", placeholderInstanceNamePrefix), instance.Name)
-	return matched
+	return strings.HasPrefix(instance.Name, placeholderInstanceNamePrefix)
 }
 
 // Fetch automatically discovered ASGs. These ASGs should be unregistered if

cluster-autoscaler/cloudprovider/aws/aws_cloud_provider.go

@@ -49,13 +49,16 @@ var (
 type awsCloudProvider struct {
 	awsManager      *AwsManager
 	resourceLimiter *cloudprovider.ResourceLimiter
+	// InstanceTypes is a map of ec2 resources
+	instanceTypes map[string]*InstanceType
 }
 
 // BuildAwsCloudProvider builds CloudProvider implementation for AWS.
-func BuildAwsCloudProvider(awsManager *AwsManager, resourceLimiter *cloudprovider.ResourceLimiter) (cloudprovider.CloudProvider, error) {
+func BuildAwsCloudProvider(awsManager *AwsManager, instanceTypes map[string]*InstanceType, resourceLimiter *cloudprovider.ResourceLimiter) (cloudprovider.CloudProvider, error) {
 	aws := &awsCloudProvider{
 		awsManager:      awsManager,
 		resourceLimiter: resourceLimiter,
+		instanceTypes:   instanceTypes,
 	}
 	return aws, nil
 }
@@ -343,12 +346,37 @@ func BuildAWS(opts config.AutoscalingOptions, do cloudprovider.NodeGroupDiscover
 		defer config.Close()
 	}
 
+	// Generate EC2 list
+	var instanceTypes map[string]*InstanceType
+	var lastUpdateTime string
+	if opts.AWSUseStaticInstanceList {
+		instanceTypes, lastUpdateTime = GetStaticEC2InstanceTypes()
+		klog.Warningf("Use static EC2 Instance Types and list could be outdated. Last update time: %s", lastUpdateTime)
+	} else {
+		region, err := GetCurrentAwsRegion()
+		if err != nil {
+			klog.Fatalf("Failed to get AWS Region: %v", err)
+		}
+
+		instanceTypes, err = GenerateEC2InstanceTypes(region)
+		if err != nil {
+			klog.Fatalf("Failed to generate AWS EC2 Instance Types: %v", err)
+		}
+
+		keys := make([]string, 0, len(instanceTypes))
+		for key := range instanceTypes {
+			keys = append(keys, key)
+		}
+
+		klog.Infof("Successfully load %d EC2 Instance Types %s", len(keys), keys)
+	}
+
 	manager, err := CreateAwsManager(config, do)
 	if err != nil {
 		klog.Fatalf("Failed to create AWS Manager: %v", err)
 	}
 
-	provider, err := BuildAwsCloudProvider(manager, rl)
+	provider, err := BuildAwsCloudProvider(manager, instanceTypes, rl)
 	if err != nil {
 		klog.Fatalf("Failed to create AWS cloud provider: %v", err)
 	}

cluster-autoscaler/cloudprovider/aws/aws_cloud_provider_test.go

@@ -79,7 +79,7 @@ var testAwsManager = &AwsManager{
 	autoScalingService: testService,
 }
 
-func newTestAwsManagerWithService(service autoScaling, autoDiscoverySpecs []cloudprovider.ASGAutoDiscoveryConfig) *AwsManager {
+func newTestAwsManagerWithService(service autoScaling, autoDiscoverySpecs []asgAutoDiscoveryConfig) *AwsManager {
 	wrapper := autoScalingWrapper{service, map[string]string{}}
 	return &AwsManager{
 		autoScalingService: wrapper,
@@ -101,7 +101,7 @@ func newTestAwsManagerWithAsgs(t *testing.T, service autoScaling, specs []string
 	return m
 }
 
-func newTestAwsManagerWithAutoAsgs(t *testing.T, service autoScaling, specs []string, autoDiscoverySpecs []cloudprovider.ASGAutoDiscoveryConfig) *AwsManager {
+func newTestAwsManagerWithAutoAsgs(t *testing.T, service autoScaling, specs []string, autoDiscoverySpecs []asgAutoDiscoveryConfig) *AwsManager {
 	m := newTestAwsManagerWithService(service, autoDiscoverySpecs)
 	m.asgCache.parseExplicitAsgs(specs)
 	return m
@@ -123,6 +123,7 @@ func testNamedDescribeAutoScalingGroupsOutput(groupName string, desiredCap int64
 				MinSize:              aws.Int64(1),
 				MaxSize:              aws.Int64(5),
 				Instances:            instances,
+				AvailabilityZones:    aws.StringSlice([]string{"us-east-1a"}),
 			},
 		},
 	}
@@ -133,7 +134,8 @@ func testProvider(t *testing.T, m *AwsManager) *awsCloudProvider {
 		map[string]int64{cloudprovider.ResourceNameCores: 1, cloudprovider.ResourceNameMemory: 10000000},
 		map[string]int64{cloudprovider.ResourceNameCores: 10, cloudprovider.ResourceNameMemory: 100000000})
 
-	provider, err := BuildAwsCloudProvider(m, resourceLimiter)
+	instanceTypes, _ := GetStaticEC2InstanceTypes()
+	provider, err := BuildAwsCloudProvider(m, instanceTypes, resourceLimiter)
 	assert.NoError(t, err)
 	return provider.(*awsCloudProvider)
 }
@@ -143,7 +145,8 @@ func TestBuildAwsCloudProvider(t *testing.T) {
 		map[string]int64{cloudprovider.ResourceNameCores: 1, cloudprovider.ResourceNameMemory: 10000000},
 		map[string]int64{cloudprovider.ResourceNameCores: 10, cloudprovider.ResourceNameMemory: 100000000})
 
-	_, err := BuildAwsCloudProvider(testAwsManager, resourceLimiter)
+	instanceTypes, _ := GetStaticEC2InstanceTypes()
+	_, err := BuildAwsCloudProvider(testAwsManager, instanceTypes, resourceLimiter)
 	assert.NoError(t, err)
 }
 
@@ -164,7 +167,7 @@ func TestNodeGroups(t *testing.T) {
 
 func TestAutoDiscoveredNodeGroups(t *testing.T) {
 	service := &AutoScalingMock{}
-	provider := testProvider(t, newTestAwsManagerWithAutoAsgs(t, service, []string{}, []cloudprovider.ASGAutoDiscoveryConfig{
+	provider := testProvider(t, newTestAwsManagerWithAutoAsgs(t, service, []string{}, []asgAutoDiscoveryConfig{
 		{
 			Tags: map[string]string{"test": ""},
 		},
@@ -437,6 +440,53 @@ func TestDeleteNodes(t *testing.T) {
 	assert.Equal(t, 1, newSize)
 }
 
+func TestDeleteNodesWithPlaceholder(t *testing.T) {
+	service := &AutoScalingMock{}
+	provider := testProvider(t, newTestAwsManagerWithAsgs(t, service, []string{"1:5:test-asg"}))
+	asgs := provider.NodeGroups()
+
+	service.On("SetDesiredCapacity", &autoscaling.SetDesiredCapacityInput{
+		AutoScalingGroupName: aws.String(asgs[0].Id()),
+		DesiredCapacity:      aws.Int64(1),
+		HonorCooldown:        aws.Bool(false),
+	}).Return(&autoscaling.SetDesiredCapacityOutput{})
+
+	// Look up the current number of instances...
+	var expectedInstancesCount int64 = 2
+	service.On("DescribeAutoScalingGroupsPages",
+		&autoscaling.DescribeAutoScalingGroupsInput{
+			AutoScalingGroupNames: aws.StringSlice([]string{"test-asg"}),
+			MaxRecords:            aws.Int64(maxRecordsReturnedByAPI),
+		},
+		mock.AnythingOfType("func(*autoscaling.DescribeAutoScalingGroupsOutput, bool) bool"),
+	).Run(func(args mock.Arguments) {
+		fn := args.Get(1).(func(*autoscaling.DescribeAutoScalingGroupsOutput, bool) bool)
+		fn(testNamedDescribeAutoScalingGroupsOutput("test-asg", expectedInstancesCount, "test-instance-id"), false)
+		// we expect the instance count to be 1 after the call to DeleteNodes
+		expectedInstancesCount = 1
+	}).Return(nil)
+
+	provider.Refresh()
+
+	initialSize, err := asgs[0].TargetSize()
+	assert.NoError(t, err)
+	assert.Equal(t, 2, initialSize)
+
+	node := &apiv1.Node{
+		Spec: apiv1.NodeSpec{
+			ProviderID: "aws:///us-east-1a/i-placeholder-test-asg-1",
+		},
+	}
+	err = asgs[0].DeleteNodes([]*apiv1.Node{node})
+	assert.NoError(t, err)
+	service.AssertNumberOfCalls(t, "SetDesiredCapacity", 1)
+	service.AssertNumberOfCalls(t, "DescribeAutoScalingGroupsPages", 2)
+
+	newSize, err := asgs[0].TargetSize()
+	assert.NoError(t, err)
+	assert.Equal(t, 1, newSize)
+}
+
 func TestDeleteNodesAfterMultipleRefreshes(t *testing.T) {
 	service := &AutoScalingMock{}
 	manager := newTestAwsManagerWithAsgs(t, service, []string{"1:5:test-asg"})