Merge pull request #241 from elmiko/merge-1.25.0

rebase on upstream 1.25.0

.github/workflows/ci.yaml

@@ -7,14 +7,17 @@ on:
 env:
   GOPATH: ${{ github.workspace }}/go
 
+permissions:
+  contents: read
+
 jobs:
   test-and-verify:
     runs-on: ubuntu-latest
     steps:
       - name: Set up Go
         uses: actions/setup-go@v2
         with:
-          go-version: 1.17.5
+          go-version: 1.18.1
 
       - uses: actions/checkout@v2
         with:

.github/workflows/pr.yaml

@@ -1,5 +1,11 @@
+permissions:
+  contents: read
+
 jobs:
   changes:
+    permissions:
+      contents: read  # for dorny/paths-filter to fetch a list of changed files
+      pull-requests: read  # for dorny/paths-filter to read pull requests
     outputs:
       charts: ${{ steps.filter.outputs.charts }}
     runs-on: ubuntu-latest

.github/workflows/release.yaml

@@ -1,5 +1,10 @@
+permissions:
+  contents: read
+
 jobs:
   release:
+    permissions:
+      contents: write  # for helm/chart-releaser-action to push chart release and create a release
     runs-on: ubuntu-latest
     steps:
       - name: Checkout

builder/Dockerfile

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM golang:1.18.1
+FROM golang:1.19
 LABEL maintainer="Marcin Wielgus <mwielgus@google.com>"
 
 ENV GOPATH /gopath/

charts/cluster-autoscaler/Chart.yaml

@@ -5,16 +5,10 @@ engine: gotpl
 home: https://github.com/kubernetes/autoscaler
 icon: https://github.com/kubernetes/kubernetes/raw/master/logo/logo.png
 maintainers:
-  - email: e.bailey@sportradar.com
-    name: yurrriq
-  - email: mgoodness@gmail.com
-    name: mgoodness
   - email: guyjtempleton@googlemail.com
     name: gjtempleton
-  - email: scott.crooks@gmail.com
-    name: sc250024
 name: cluster-autoscaler
 sources:
   - https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler
 type: application
-version: 9.18.1
+version: 9.20.1

charts/cluster-autoscaler/README.md

@@ -287,7 +287,7 @@ Though enough for the majority of installations, the default PodSecurityPolicy _
 |-----|------|---------|-------------|
 | additionalLabels | object | `{}` | Labels to add to each object of the chart. |
 | affinity | object | `{}` | Affinity for pod assignment |
-| autoDiscovery.clusterName | string | `nil` | Enable autodiscovery for `cloudProvider=aws`, for groups matching `autoDiscovery.tags`. Enable autodiscovery for `cloudProvider=clusterapi`, for groups matching `autoDiscovery.labels`. Enable autodiscovery for `cloudProvider=gce`, but no MIG tagging required. Enable autodiscovery for `cloudProvider=magnum`, for groups matching `autoDiscovery.roles`. |
+| autoDiscovery.clusterName | string | `nil` | Enable autodiscovery for `cloudProvider=aws`, for groups matching `autoDiscovery.tags`. autoDiscovery.clusterName -- Enable autodiscovery for `cloudProvider=azure`, using tags defined in https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/azure/README.md#auto-discovery-setup. Enable autodiscovery for `cloudProvider=clusterapi`, for groups matching `autoDiscovery.labels`. Enable autodiscovery for `cloudProvider=gce`, but no MIG tagging required. Enable autodiscovery for `cloudProvider=magnum`, for groups matching `autoDiscovery.roles`. |
 | autoDiscovery.labels | list | `[]` | Cluster-API labels to match  https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/clusterapi/README.md#configuring-node-group-auto-discovery |
 | autoDiscovery.roles | list | `["worker"]` | Magnum node group roles to match. |
 | autoDiscovery.tags | list | `["k8s.io/cluster-autoscaler/enabled","k8s.io/cluster-autoscaler/{{ .Values.autoDiscovery.clusterName }}"]` | ASG tags to match, run through `tpl`. |
@@ -305,7 +305,7 @@ Though enough for the majority of installations, the default PodSecurityPolicy _
 | azureTenantID | string | `""` | Azure tenant where the resources are located. Required if `cloudProvider=azure` |
 | azureUseManagedIdentityExtension | bool | `false` | Whether to use Azure's managed identity extension for credentials. If using MSI, ensure subscription ID, resource group, and azure AKS cluster name are set. |
 | azureVMType | string | `"AKS"` | Azure VM type. |
-| cloudConfigPath | string | `"/etc/gce.conf"` | Configuration file for cloud provider. |
+| cloudConfigPath | string | `""` | Configuration file for cloud provider. |
 | cloudProvider | string | `"aws"` | The cloud provider where the autoscaler runs. Currently only `gce`, `aws`, `azure`, `magnum` and `clusterapi` are supported. `aws` supported for AWS. `gce` for GCE. `azure` for Azure AKS. `magnum` for OpenStack Magnum, `clusterapi` for Cluster API. |
 | clusterAPICloudConfigPath | string | `"/etc/kubernetes/mgmt-kubeconfig"` | Path to kubeconfig for connecting to Cluster API Management Cluster, only used if `clusterAPIMode=kubeconfig-kubeconfig or incluster-kubeconfig` |
 | clusterAPIConfigMapsNamespace | string | `""` | Namespace on the workload cluster to store Leader election and status configmaps |
@@ -356,13 +356,15 @@ Though enough for the majority of installations, the default PodSecurityPolicy _
 | resources | object | `{}` | Pod resource requests and limits. |
 | securityContext | object | `{}` | [Security context for pod](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/) |
 | service.annotations | object | `{}` | Annotations to add to service |
+| service.create | bool | `true` | If `true`, a Service will be created. |
 | service.externalIPs | list | `[]` | List of IP addresses at which the service is available. Ref: https://kubernetes.io/docs/user-guide/services/#external-ips. |
 | service.labels | object | `{}` | Labels to add to service |
 | service.loadBalancerIP | string | `""` | IP address to assign to load balancer (if supported). |
 | service.loadBalancerSourceRanges | list | `[]` | List of IP CIDRs allowed access to load balancer (if supported). |
 | service.portName | string | `"http"` | Name for service port. |
 | service.servicePort | int | `8085` | Service port to expose. |
 | service.type | string | `"ClusterIP"` | Type of service to create. |
+| serviceMonitor.annotations | object | `{}` | Annotations to add to service monitor |
 | serviceMonitor.enabled | bool | `false` | If true, creates a Prometheus Operator ServiceMonitor. |
 | serviceMonitor.interval | string | `"10s"` | Interval that Prometheus scrapes Cluster Autoscaler metrics. |
 | serviceMonitor.namespace | string | `"monitoring"` | Namespace which Prometheus is running in. |

charts/cluster-autoscaler/templates/deployment.yaml

@@ -63,6 +63,9 @@ spec:
             {{- if .Values.autoDiscovery.clusterName }}
             - --node-group-auto-discovery=asg:tag={{ tpl (join "," .Values.autoDiscovery.tags) . }}
             {{- end }}
+            {{- if .Values.cloudConfigPath }}
+            - --cloud-config={{ .Values.cloudConfigPath }}
+            {{- end }}
           {{- else if eq .Values.cloudProvider "gce" }}
           {{- if .Values.autoscalingGroupsnamePrefix }}
             {{- range .Values.autoscalingGroupsnamePrefix }}
@@ -91,6 +94,10 @@ spec:
             {{- else if eq .Values.clusterAPIMode "single-kubeconfig"}}
             - --kubeconfig={{ .Values.clusterAPIWorkloadKubeconfigPath }}
             {{- end }}
+          {{- else if eq .Values.cloudProvider "azure" }}
+            {{- if .Values.autoDiscovery.clusterName }}
+            - --node-group-auto-discovery=label:cluster-autoscaler-enabled=true,cluster-autoscaler-name={{ .Values.autoDiscovery.clusterName }}
+            {{- end }}
           {{- end }}
           {{- if eq .Values.cloudProvider "magnum" }}
             - --cloud-config={{ .Values.cloudConfigPath }}

charts/cluster-autoscaler/templates/podsecuritypolicy.yaml

@@ -19,10 +19,6 @@ spec:
     - 'emptyDir'
     - 'projected'
     - 'downwardAPI'
-{{- if eq .Values.cloudProvider "gce" }}
-  allowedHostPaths:
-    - pathPrefix: {{ .Values.cloudConfigPath }}
-{{- end }}
   hostNetwork: false
   hostIPC: false
   hostPID: false

charts/cluster-autoscaler/templates/priority-expander-configmap.yaml

@@ -16,7 +16,10 @@ data:
 {{- if kindIs "string" .Values.expanderPriorities }}
 {{ .Values.expanderPriorities | indent 4 }}
 {{- else }}
-{{ toYaml .Values.expanderPriorities | indent 4 }}
+{{- range $k,$v := .Values.expanderPriorities }}
+    {{ $k | int }}:
+      {{- toYaml $v | nindent 6 }}
+{{- end -}}
 {{- end -}}
 {{- end -}}
 {{- end -}}

charts/cluster-autoscaler/templates/service.yaml

@@ -1,3 +1,4 @@
+{{- if .Values.service.create }}
 apiVersion: v1
 kind: Service
 metadata:
@@ -35,3 +36,4 @@ spec:
   selector:
 {{ include "cluster-autoscaler.instance-name" . | indent 4 }}
   type: "{{ .Values.service.type }}"
+{{- end }}

charts/cluster-autoscaler/templates/servicemonitor.yaml

@@ -6,6 +6,8 @@ metadata:
   {{- if .Values.serviceMonitor.namespace }}
   namespace: {{ .Values.serviceMonitor.namespace }}
   {{- end }}
+  annotations:
+{{ toYaml .Values.serviceMonitor.annotations | indent 4 }}
   labels:
   {{- range $key, $value := .Values.serviceMonitor.selector }}
     {{ $key }}: {{ $value | quote }}

charts/cluster-autoscaler/values.yaml

@@ -3,10 +3,11 @@
 affinity: {}
 
 autoDiscovery:
-  # cloudProviders `aws`, `gce`, `magnum` and `clusterapi` are supported by auto-discovery at this time
+  # cloudProviders `aws`, `gce`, `azure`, `magnum` and `clusterapi` are supported by auto-discovery at this time
   # AWS: Set tags as described in https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws/README.md#auto-discovery-setup
 
   # autoDiscovery.clusterName -- Enable autodiscovery for `cloudProvider=aws`, for groups matching `autoDiscovery.tags`.
+  # autoDiscovery.clusterName -- Enable autodiscovery for `cloudProvider=azure`, using tags defined in https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/azure/README.md#auto-discovery-setup.
   # Enable autodiscovery for `cloudProvider=clusterapi`, for groups matching `autoDiscovery.labels`.
   # Enable autodiscovery for `cloudProvider=gce`, but no MIG tagging required.
   # Enable autodiscovery for `cloudProvider=magnum`, for groups matching `autoDiscovery.roles`.
@@ -124,7 +125,7 @@ clusterAPICloudConfigPath: /etc/kubernetes/mgmt-kubeconfig
 clusterAPIConfigMapsNamespace: ""
 
 # cloudConfigPath -- Configuration file for cloud provider.
-cloudConfigPath: /etc/gce.conf
+cloudConfigPath: ""
 
 # cloudProvider -- The cloud provider where the autoscaler runs.
 # Currently only `gce`, `aws`, `azure`, `magnum` and `clusterapi` are supported.
@@ -306,6 +307,8 @@ securityContext: {}
   # runAsGroup: 1001
 
 service:
+  # service.create -- If `true`, a Service will be created.
+  create: true
   # service.annotations -- Annotations to add to service
   annotations: {}
   # service.labels -- Labels to add to service
@@ -339,6 +342,8 @@ serviceMonitor:
     release: prometheus-operator
   # serviceMonitor.path -- The path to scrape for metrics; autoscaler exposes `/metrics` (this is standard)
   path: /metrics
+  # serviceMonitor.annotations -- Annotations to add to service monitor
+  annotations: {}
 
 ## Custom PrometheusRule to be defined
 ## The value is evaluated as a template, so, for example, the value can depend on .Release or .Chart

cluster-autoscaler/Dockerfile.amd64

@@ -11,9 +11,10 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-ARG BASEIMAGE=gcr.io/distroless/static:latest-amd64
+ARG BASEIMAGE=gcr.io/distroless/static:nonroot-amd64
 FROM $BASEIMAGE
 LABEL maintainer="Marcin Wielgus <mwielgus@google.com>"
 
 COPY cluster-autoscaler-amd64 /cluster-autoscaler
+WORKDIR /
 CMD ["/cluster-autoscaler"]

cluster-autoscaler/Dockerfile.arm64

@@ -11,9 +11,10 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-ARG BASEIMAGE=gcr.io/distroless/static:latest-arm64
+ARG BASEIMAGE=gcr.io/distroless/static:nonroot-arm64
 FROM $BASEIMAGE
 LABEL maintainer="Marcin Wielgus <mwielgus@google.com>"
 
 COPY cluster-autoscaler-arm64 /cluster-autoscaler
+WORKDIR /
 CMD ["/cluster-autoscaler"]

cluster-autoscaler/FAQ.md

@@ -28,6 +28,7 @@ this document:
 * [How to?](#how-to)
   * [I'm running cluster with nodes in multiple zones for HA purposes. Is that supported by Cluster Autoscaler?](#im-running-cluster-with-nodes-in-multiple-zones-for-ha-purposes-is-that-supported-by-cluster-autoscaler)
   * [How can I monitor Cluster Autoscaler?](#how-can-i-monitor-cluster-autoscaler)
+  * [How can I see all the events from Cluster Autoscaler?](#how-can-i-see-all-events-from-cluster-autoscaler)
   * [How can I scale my cluster to just 1 node?](#how-can-i-scale-my-cluster-to-just-1-node)
   * [How can I scale a node group to 0?](#how-can-i-scale-a-node-group-to-0)
   * [How can I prevent Cluster Autoscaler from scaling down a particular node?](#how-can-i-prevent-cluster-autoscaler-from-scaling-down-a-particular-node)
@@ -267,6 +268,16 @@ respectively under `/metrics` and `/health-check`.
 Metrics are provided in Prometheus format and their detailed description is
 available [here](https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/proposals/metrics.md).
 
+### How can I see all events from Cluster Autoscaler?
+
+By default, the Cluster Autoscaler will deduplicate similar events that occur within a 5 minute
+window. This is done to improve scalability performance where many similar events might be
+triggered in a short timespan, such as when there are too many unscheduled pods.
+
+In some cases, such as for debugging or when scalability of events is not an issue, you might
+want to see all the events coming from the Cluster Autoscaler. In these scenarios you should
+use the `--record-duplicated-events` command line flag.
+
 ### How can I scale my cluster to just 1 node?
 
 Prior to version 0.6, Cluster Autoscaler was not touching nodes that were running important
@@ -408,6 +419,7 @@ spec:
         run: overprovisioning
     spec:
       priorityClassName: overprovisioning
+      terminationGracePeriodSeconds: 0
       containers:
       - name: reserve-resources
         image: k8s.gcr.io/pause
@@ -433,10 +445,10 @@ spec:
         app: overprovisioning-autoscaler
     spec:
       containers:
-        - image: k8s.gcr.io/cluster-proportional-autoscaler-amd64:1.1.2
+        - image: k8s.gcr.io/cluster-proportional-autoscaler-amd64:1.8.1
           name: autoscaler
           command:
-            - ./cluster-proportional-autoscaler
+            - /cluster-proportional-autoscaler
             - --namespace=default
             - --configmap=overprovisioning-autoscaler
             - --default-params={"linear":{"coresPerReplica":1}}
@@ -720,7 +732,7 @@ The following startup parameters are supported for cluster autoscaler:
 | `scale-down-candidates-pool-min-count` | Minimum number of nodes that are considered as additional non empty candidates<br>for scale down when some candidates from previous iteration are no longer valid.<br>When calculating the pool size for additional candidates we take<br>`max(#nodes * scale-down-candidates-pool-ratio, scale-down-candidates-pool-min-count)` | 50
 | `scan-interval` | How often cluster is reevaluated for scale up or down | 10 seconds
 | `max-nodes-total` | Maximum number of nodes in all node groups. Cluster autoscaler will not grow the cluster beyond this number. | 0
-| `cores-total` | Minimum and maximum number of cores in cluster, in the format <min>:<max>. Cluster autoscaler will not scale the cluster beyond these numbers. | 320000
+| `cores-total` | Minimum and maximum number of cores in cluster, in the format \<min>:\<max>. Cluster autoscaler will not scale the cluster beyond these numbers. | 320000
 | `memory-total` | Minimum and maximum number of gigabytes of memory in cluster, in the format \<min>:\<max>. Cluster autoscaler will not scale the cluster beyond these numbers. | 6400000
 | `gpu-total` | Minimum and maximum number of different GPUs in cluster, in the format <gpu_type>:\<min>:\<max>. Cluster autoscaler will not scale the cluster beyond these numbers. Can be passed multiple times. CURRENTLY THIS FLAG ONLY WORKS ON GKE. | ""
 | `cloud-provider` | Cloud provider type. | gce
@@ -742,6 +754,7 @@ The following startup parameters are supported for cluster autoscaler:
 | `max-failing-time` | Maximum time from last recorded successful autoscaler run before automatic restart | 15 minutes
 | `balance-similar-node-groups` | Detect similar node groups and balance the number of nodes between them | false
 | `balancing-ignore-label` | Define a node label that should be ignored when considering node group similarity. One label per flag occurrence. | ""
+| `balancing-label` | Define a node label to use when comparing node group similarity. If set, all other comparison logic is disabled, and only labels are considered when comparing groups. One label per flag occurrence. | ""
 | `node-autoprovisioning-enabled` | Should CA autoprovision node groups when needed | false
 | `max-autoprovisioned-node-group-count` | The maximum number of autoprovisioned groups in the cluster | 15
 | `unremovable-node-recheck-timeout` | The timeout before we check again a node that couldn't be removed before | 5 minutes
@@ -760,6 +773,7 @@ The following startup parameters are supported for cluster autoscaler:
 | `daemonset-eviction-for-occupied-nodes` | Whether DaemonSet pods will be gracefully terminated from non-empty nodes | true
 | `feature-gates` | A set of key=value pairs that describe feature gates for alpha/experimental features. | ""
 | `cordon-node-before-terminating` | Should CA cordon nodes before terminating during downscale process | false
+| `record-duplicated-events` | Enable the autoscaler to print duplicated events within a 5 minute window. | false
 
 # Troubleshooting:
 

cluster-autoscaler/OWNERS

@@ -2,11 +2,9 @@
 
 approvers:
 - elmiko
-- alexander-demichev
 - JoelSpeed
 reviewers:
 - enxebre
 - frobware
 - elmiko
-- alexander-demichev
 - JoelSpeed

cluster-autoscaler/README.md

@@ -32,6 +32,9 @@ You should also take a look at the notes and "gotchas" for your specific cloud p
 * [BizflyCloud](./cloudprovider/bizflycloud/README.md)
 * [Vultr](./cloudprovider/vultr/README.md)
 * [TencentCloud](./cloudprovider/tencentcloud/README.md)
+* [Scaleway](./cloudprovider/scaleway/README.md)
+* [Rancher](./cloudprovider/rancher/README.md)
+* [Kamatera](./cloudprovider/kamatera/README.md)
 
 # Releases
 
@@ -177,3 +180,7 @@ Supported cloud providers:
 * Cluster API https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/clusterapi/README.md
 * Vultr https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/vultr/README.md
 * TencentCloud https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/tencentcloud/README.md
+* BaiduCloud https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/baiducloud/README.md
+* Huawei Cloud https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/huaweicloud/README.md
+* Rancher https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/rancher/README.md
+* Kamatera https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/kamatera/README.md

cluster-autoscaler/cloudprovider/alicloud/alicloud_auto_scaling.go

@@ -30,7 +30,7 @@ const (
 	defaultAdjustmentType   = "TotalCapacity"
 )
 
-//autoScaling define the interface usage in alibaba-cloud-sdk-go.
+// autoScaling define the interface usage in alibaba-cloud-sdk-go.
 type autoScaling interface {
 	DescribeScalingGroups(req *ess.DescribeScalingGroupsRequest) (*ess.DescribeScalingGroupsResponse, error)
 	DescribeScalingConfigurations(req *ess.DescribeScalingConfigurationsRequest) (*ess.DescribeScalingConfigurationsResponse, error)
@@ -95,7 +95,7 @@ func getEssClient(cfg *cloudConfig) (client *ess.Client, err error) {
 	return
 }
 
-//autoScalingWrapper will serve as the
+// autoScalingWrapper will serve as the
 type autoScalingWrapper struct {
 	autoScaling
 	cfg *cloudConfig

cluster-autoscaler/cloudprovider/alicloud/examples/cluster-autoscaler-standard.yaml

@@ -131,7 +131,7 @@ spec:
         app: cluster-autoscaler
     spec:
       priorityClassName: system-cluster-critical
-      serviceAccountName: admin
+      serviceAccountName: cluster-autoscaler
       containers:
         - image: registry.cn-hangzhou.aliyuncs.com/acs/autoscaler:v1.3.1
           name: cluster-autoscaler
@@ -162,9 +162,9 @@ spec:
                 key: access-key-secret
           - name: REGION_ID
             valueFrom:
-            secretKeyRef:
-              name: cloud-config
-              key: region-id
+              secretKeyRef:
+                name: cloud-config
+                key: region-id
           volumeMounts:
             - name: ssl-certs
               mountPath: /etc/ssl/certs/ca-certificates.crt