Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Integrate enhanced meltdown handling for dependency-watchdog probe #5497

Merged
merged 4 commits into from
Mar 8, 2022
Merged

Integrate enhanced meltdown handling for dependency-watchdog probe #5497

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
4e69680
Vendored dependency-watchdog 0.7.0
ashwani2k Mar 1, 2022
6e52ca2
Adapted dependency-watchdog component to work with DWD v0.7.0
ashwani2k Mar 1, 2022
2d60faf
Updated charts for DWD and adapted for MCM deployment under v1beta1 c…
ashwani2k Mar 1, 2022
df84f18
Adapted RBAC for dependency-watchdog-endpoint as well
ashwani2k Mar 8, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
4 changes: 2 additions & 2 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ require (
github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021
github.com/frankban/quicktest v1.13.1 // indirect
github.com/gardener/component-spec/bindings-go v0.0.33
github.com/gardener/dependency-watchdog v0.6.1-0.20210623112844-96f73d5dc311
github.com/gardener/dependency-watchdog v0.7.0
ialidzhikov marked this conversation as resolved.
Show resolved Hide resolved
github.com/gardener/etcd-druid v0.7.0
github.com/gardener/external-dns-management v0.7.18
github.com/gardener/hvpa-controller v0.3.1
Expand Down Expand Up @@ -45,7 +45,7 @@ require (
go.uber.org/zap v1.19.1
golang.org/x/crypto v0.0.0-20210817164053-32db794688a5
golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac
golang.org/x/tools v0.1.7
golang.org/x/tools v0.1.9
gomodules.xyz/jsonpatch/v2 v2.2.0
gonum.org/v1/gonum v0.8.2
google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368 // indirect
Expand Down
14 changes: 9 additions & 5 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -233,8 +233,8 @@ github.com/gardener/component-spec/bindings-go v0.0.33/go.mod h1:53EIwICsuMgbY/M
github.com/gardener/controller-manager-library v0.1.1-0.20191212112146-917449ad760c/go.mod h1:v6cbldxmpL2fYBEB2lSnq3LSEPwIHus9En6iIhwNE1k=
github.com/gardener/controller-manager-library v0.1.1-0.20200204110458-c263b9bb97ad/go.mod h1:v6cbldxmpL2fYBEB2lSnq3LSEPwIHus9En6iIhwNE1k=
github.com/gardener/controller-manager-library v0.2.1-0.20200810091329-d980dbe10959/go.mod h1:XMp1tPcX3SP/dMd+3id418f5Cqu44vydeTkBRbW8EvQ=
github.com/gardener/dependency-watchdog v0.6.1-0.20210623112844-96f73d5dc311 h1:UaBOYSCJkISXUlKiSwHSiHAM+4yotKvkd6GQMkC4ke4=
github.com/gardener/dependency-watchdog v0.6.1-0.20210623112844-96f73d5dc311/go.mod h1:h4miVhg/rGUl8jZsmRrunKDX56I6gHvr2HtC7gBepIY=
github.com/gardener/dependency-watchdog v0.7.0 h1:54Hy9XxQrP8j0bpzJo9k5tJ3Nd/snRX+lCZH/aYdhG8=
github.com/gardener/dependency-watchdog v0.7.0/go.mod h1:OxGZPFb0z8/dTVsk0dCIgcF+q9JhXvfVIdzpYOkcnl8=
github.com/gardener/etcd-druid v0.1.12/go.mod h1:yZrUQY9clD8/ZXK+MmEq8OS1TaKJeipV0u4kHHrwWeY=
github.com/gardener/etcd-druid v0.1.15/go.mod h1:BHXG8N04Dl4On7Ie6cErwmpvzncNrmeb+HO7Sqrhf+A=
github.com/gardener/etcd-druid v0.3.0/go.mod h1:uxZjZ57gIgpX554vGp495g2i8DByoS3OkVtiqsxtbwk=
Expand Down Expand Up @@ -808,6 +808,7 @@ github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU=
go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4=
Expand Down Expand Up @@ -948,8 +949,9 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo=
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.5.1 h1:OJxoQ/rynoF0dcCdI7cLPktw/hR2cueqYfjm43oqK38=
golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro=
golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180112015858-5ccada7d0a7b/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
Expand Down Expand Up @@ -1001,6 +1003,7 @@ golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT
golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211209124913-491a49abca63 h1:iocB37TsdFuN6IBRZ+ry36wrkoV51/tl5vOWqkcPGvY=
golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
Expand Down Expand Up @@ -1109,6 +1112,7 @@ golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211029165221-6e7872819dc8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM=
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
Expand Down Expand Up @@ -1211,8 +1215,8 @@ golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/tools v0.1.6-0.20210820212750-d4cc65f0b2ff/go.mod h1:YD9qOF0M9xpSpdWTBbzEl5e/RnCefISl8E5Noe10jFM=
golang.org/x/tools v0.1.7 h1:6j8CgantCy3yc8JGBqkDLMKWqZ0RDU2g1HVgacojGWQ=
golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo=
golang.org/x/tools v0.1.9 h1:j9KsMiaP1c3B0OTQGth0/k+miLGTgLsAFUCrF2vLcF8=
golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
Expand Down
11 changes: 11 additions & 0 deletions pkg/operation/botanist/component/dependencywatchdog/dependency_watchdog.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -362,6 +362,17 @@ func (d *dependencyWatchdog) clusterRoleRules() []rbacv1.PolicyRule {
Resources: []string{"deployments", "deployments/scale"},
Verbs: []string{"get", "list", "watch", "update"},
},
{
APIGroups: []string{"coordination.k8s.io"},
Resources: []string{"leases"},
Verbs: []string{"create"},
},
{
APIGroups: []string{"coordination.k8s.io"},
ResourceNames: []string{"dependency-watchdog-probe"},
ialidzhikov marked this conversation as resolved.
Show resolved Hide resolved
Resources: []string{"leases"},
Verbs: []string{"get", "watch", "update"},
},
}
}

Expand Down
16 changes: 16 additions & 0 deletions pkg/operation/botanist/component/dependencywatchdog/dependency_watchdog_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,22 @@ rules:`
- list
- watch
- update
- apiGroups:
- coordination.k8s.io
resources:
- leases
verbs:
- create
- apiGroups:
- coordination.k8s.io
resourceNames:
- dependency-watchdog-probe
resources:
- leases
verbs:
- get
- watch
- update
`
}

Expand Down
45 changes: 39 additions & 6 deletions pkg/operation/botanist/component/kubeapiserver/dependency_watchdog.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
package kubeapiserver

import (
worker "github.com/gardener/gardener/extensions/pkg/controller/worker/genericactuator"
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why we are importing a pkg from the extension library for a constant usage? I guess this is also why make verify fails (import-boss not allowing to import the extension library from ./pkg).

Copy link
Contributor Author

@ashwani2k ashwani2k Mar 1, 2022 •
edited

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I couldn't find any constant in g/g for this in v1beta1constants. So I used the one from the extension as I didn't want to introduce a new one if it goes untracked.
However as suggested by @rfranzke in #5497 (comment) to add it in v1beta1constants.
So this is fixed with the commit

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

With GEP-01 (extensibility) cloudprovider specific details are extracted to extensions. gardenlet does not need to know anything about MCM or cannot make any assumption that MCM is used. Actually from gardenlet's point of view there is only the Worker resource and that is the contract. The fact that provider extensions choose to deploy MCM as part of the Worker reconciliation is not a thing that gardenlet has to assume. IMO this PR is violating GEP-01 as it is making gardenlet to configure dependency-watchdog assuming that MCM is used. In theory, a provider extension can implement the contract without using MCM.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@ialidzhikov See #5497 (comment)

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I agree to your concern @ialidzhikov.
Do you suggest an alternate approach here or you are fine with adding MCM deployment name in the constants.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Looks like it is hard to find out something with the new scaleRefDependsOn approach. Generally, having in mind the old config I was thinking of a well-known label that is configured in the dependency-watchdog-probe scales down all Deployments that match the well-known label - this allows extensions using MCM to add the well-known label to the MCM Deployment and in this way to "request" the MCM to be scaled down.
Do we need actually scaleRefDependsOn? Can't we simply scale down all components when the probe fails?
Assuming that the scaleRefDependsOn handling is needed, I am "fine" with the current approach because I cannot think of a good alternative.

Copy link
Contributor Author

@ashwani2k ashwani2k Mar 2, 2022 •
edited

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We can think of it. Currently I didn't want to introduce new semantics of identifying a scale resources. As the implementation already requires us to provide MCM as part of scaleRef as mentioned below

scaleRef:
       apiVersion: apps/v1
       kind: Deployment
       name: machine-controller-manager

So scaleRefDependsOn is not the issue here, if we want to do what you suggest we also need to change the design for scaleRef itself to have a new approach for selecting the deployment.

Do we need actually scaleRefDependsOn? Can't we simply scale down all components when the probe fails?

The logic currently works seamless for both ScaleUp and ScaleDowns.
Like what you mentioned scalingDown is done all at once. But, we need to consider the semantics of ScaleUp here. This is where we have a problem today, we scale up everything at once which is detrimental to even introduce MCM additionally along with KCM. As once MCM comes ups it will mark the nodes as Unknown before giving KCM any chance to update the node status and will start removing them. To avoid this we wish to delay it using scaleUpDelaySeconds.
However, even then we run the risk of KCM not being available and just starting MCM with some delay without checking if KCM is up will lead us to the same issue. So we introduced the scaleRefDependOn semantics to avoid running MCM on a state of the system which is not yet updated by KCM.
To not reinvent the wheel scaleRefDependsOn is just an array of same type as scaleRef.

We will explore if there is a better way to do it without breaking the extension contract for Gardener.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I again suggest to be pragmatic here and rather focus on getting this change in so that we can move on the other (blocking) issues like gardener/dependency-watchdog#36. IMO it's not problematic to specify MCM here as explained above.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We will explore if there is a better way to do it without breaking the extension contract for Gardener.

However, can you create an issue on dependency-watchdog side to make sure that we don't forget about this and this item is considered/worked on/brainstormed on when there is capacity? Thanks in advance!

Copy link
Contributor Author

@ashwani2k ashwani2k Mar 8, 2022 •
edited

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@ialidzhikov As suggested created an Issue on DWD to track this.

v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants"

restarterapi "github.com/gardener/dependency-watchdog/pkg/restarter/api"
Expand Down Expand Up @@ -70,12 +71,44 @@ func DependencyWatchdogProbeConfiguration() ([]scalerapi.ProbeDependants, error)
Internal: &scalerapi.ProbeDetails{KubeconfigSecretName: DependencyWatchdogInternalProbeSecretName},
PeriodSeconds: pointer.Int32(30),
},
DependantScales: []*scalerapi.DependantScaleDetails{{
ScaleRef: autoscalingv1.CrossVersionObjectReference{
APIVersion: appsv1.SchemeGroupVersion.String(),
Kind: "Deployment",
Name: v1beta1constants.DeploymentNameKubeControllerManager,
DependantScales: []*scalerapi.DependantScaleDetails{
{
ScaleRef: autoscalingv1.CrossVersionObjectReference{
APIVersion: appsv1.SchemeGroupVersion.String(),
Kind: "Deployment",
Name: v1beta1constants.DeploymentNameKubeControllerManager,
},
ScaleUpDelaySeconds: pointer.Int32(120),
},
{
ScaleRef: autoscalingv1.CrossVersionObjectReference{
APIVersion: appsv1.SchemeGroupVersion.String(),
Kind: "Deployment",
Name: worker.McmDeploymentName,
},
ScaleUpDelaySeconds: pointer.Int32(60),
ScaleRefDependsOn: []autoscalingv1.CrossVersionObjectReference{
{
APIVersion: appsv1.SchemeGroupVersion.String(),
Kind: "Deployment",
Name: v1beta1constants.DeploymentNameKubeControllerManager,
},
},
},
}},
{
ScaleRef: autoscalingv1.CrossVersionObjectReference{
APIVersion: appsv1.SchemeGroupVersion.String(),
Kind: "Deployment",
Name: v1beta1constants.DeploymentNameClusterAutoscaler,
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What happens if the Shoot does not enable autoscaling -> the cluster-autoscaler deployment is not present?

E0304 07:15:46.904892       1 prober.go:405] Scaling up dependents of shoot-kube-apiserver/shoot--foo--bar: apps/v1.Deployment/cluster-autoscaler: replicas=1: failed
E0304 07:15:46.906923       1 prober.go:452] Scaling up dependents of shoot-kube-apiserver/shoot--foo--bar: apps/v1.Deployment/cluster-autoscaler: error getting deployments.apps: deployments/scale.apps "cluster-autoscaler" not found
E0304 07:15:46.906938       1 prober.go:500] Scaling up dependents of shoot-kube-apiserver/shoot--foo--bar: apps/v1.Deployment/cluster-autoscaler: Could not get target reference: deployments/scale.apps "cluster-autoscaler" not found
E0304 07:15:46.906943       1 prober.go:501] Scaling up dependents of shoot-kube-apiserver/shoot--foo--bar: apps/v1.Deployment/cluster-autoscaler: replicas=1: failed

1 drawback I see is that in this case the logs are "polluted" with such error logs. And this is only 1 Shoot, image 50 Shoots on a Seed to have cluster-autoscaler disabled. dependency-watchdog should rather know that this component is optional and log in info level something like "the Deployment is not present, hence skipping it".

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If this does not cause issues for the dependency-watchdog working, then I am also fine to create an issue on dependency-watchdog side about the verbose error logging and hope that this can be fixed with an upcoming version of the component.

Copy link
Contributor Author

@ashwani2k ashwani2k Mar 7, 2022 •
edited

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

A totally valid concern. I've filed an Issue with the DWD repo. Also filed a PR for the fix and it also vendor Go 1.17 to be part of the patch release. v0.7.1

The new logs when the cluster-autoscaler is available will be:

| I0307 15:11:17.440689       1 prober.go:368] shoot-kube-apiserver/shoot--<project>--<shootname>/external: probe result: &scaler.probeResult{lastError:(*url.Error)(0xc000523200), resultRun:4}                                                                                                                                                                                  │
│ I0307 15:11:17.440816       1 prober.go:414] Scaling down dependents of shoot-kube-apiserver/shoot--<project>--<shootname>: apps/v1.Deployment/kube-controller-manager: skipped because desired=0 and current=0                                                                                                                                                                 │
│ I0307 15:11:17.441299       1 prober.go:414] Scaling down dependents of shoot-kube-apiserver/shoot--<project>--<shootname>: apps/v1.Deployment/machine-controller-manager: skipped because desired=0 and current=0                                                                                                                                                              │
│ I0307 15:11:17.441319       1 prober.go:414] Scaling down dependents of shoot-kube-apiserver/shoot--<project>--<shootname>: apps/v1.Deployment/cluster-autoscaler: skipped because desired=0 and current=0                                                                                                                                                                      │
│ I0307 15:11:37.582672       1 reflector.go:268] github.com/gardener/gardener/pkg/client/extensions/informers/externalversions/factory.go:117: forcing resync                                                                                                                                                                                                             │

When cluster-autoscaler deployment is not present will be:

│ I0307 15:11:37.582763       1 scaler.go:67] Update event on cluster: shoot--<project>--<shootname>                                                                                                                                                                                                                                                                              │
│ I0307 15:11:38.692669       1 reflector.go:268] k8s.io/client-go/informers/factory.go:135: forcing resync                                                                                                                                                                                                                                                                │
│ I0307 15:11:51.013434       1 prober.go:353] shoot-kube-apiserver/shoot--<project>--<shootname>/internal: probe succeeded                                                                                                                                                                                                                                                       │
│ I0307 15:11:51.013449       1 prober.go:368] shoot-kube-apiserver/shoot--<project>--<shootname>/internal: probe result: &scaler.probeResult{lastError:error(nil), resultRun:4}                                                                                                                                                                                                  │
│ I0307 15:11:51.024917       1 prober.go:356] shoot-kube-apiserver/shoot--<project>--<shootname>/external: probe failed with error: Get "https://api.<shootname>.<cluster-address>.com/version?timeout=10s": dial tcp: lookup api.<shootname>.cluster-address>.com on 100.64.0.10:53: no such host. Will retry...                                       │
│ I0307 15:11:51.024987       1 prober.go:368] shoot-kube-apiserver/shoot--<project>--<shootname>/external: probe result: &scaler.probeResult{lastError:(*url.Error)(0xc0005a1230), resultRun:4}                                                                                                                                                                                  │
│ I0307 15:11:51.025030       1 prober.go:414] Scaling down dependents of shoot-kube-apiserver/shoot--<project>--<shootname>: apps/v1.Deployment/kube-controller-manager: skipped because desired=0 and current=0                                                                                                                                                                 │
│ I0307 15:11:51.025280       1 prober.go:414] Scaling down dependents of shoot-kube-apiserver/shoot--<project>--<shootname>: apps/v1.Deployment/machine-controller-manager: skipped because desired=0 and current=0                                                                                                                                                              │
│ E0307 15:11:51.025351       1 prober.go:405] Scaling down dependents of shoot-kube-apiserver/shoot--<project>--<shootname>: apps/v1.Deployment/cluster-autoscaler: Skipped as target reference: deployment.apps "cluster-autoscaler" not found                                                                                                                                  │
│ I0307 15:12:07.583315       1 reflector.go:268] github.com/gardener/gardener/pkg/client/extensions/informers/externalversions/factory.go:117: forcing resync                                                                                                                                                                                                             │
│ I0307 15:12:07.583545       1 scaler.go:67] Update event on cluster: shoot--<project>--<shootname>

Once its merged I can also cut a patch release and merge it here or vendor it later if it takes a lot more time to merge.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks for the follow-up. Should we wait for dependency-watchdog@v0.7.1 as part of this PR or should we proceed with dependency-watchdog@v0.7.0?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'll check with @shreyas-s-rao if we can merge today and release a patch. In that case we can go with v0.7.1.
If we can not release it today, then you can go ahead and we vendor it along with the bug fix for secret rotation issue which will require a patch release.

Copy link
Contributor Author

@ashwani2k ashwani2k Mar 8, 2022 •
edited

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@ialidzhikov I checked however we won't be able to cut the release today, as Shreyas won't be able to complete the review today due to other things at hand. So either we wait till tomorrow as we are confident of releasing it tomorrow or we can vendor it with the next set of changes for DWD.

},
ScaleRefDependsOn: []autoscalingv1.CrossVersionObjectReference{
{
APIVersion: appsv1.SchemeGroupVersion.String(),
Kind: "Deployment",
Name: worker.McmDeploymentName,
},
},
},
},
}}, nil
}
45 changes: 39 additions & 6 deletions pkg/operation/botanist/component/kubeapiserver/dependency_watchdog_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import (
scalerapi "github.com/gardener/dependency-watchdog/pkg/scaler/api"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
appsv1 "k8s.io/api/apps/v1"
autoscalingv1 "k8s.io/api/autoscaling/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/utils/pointer"
Expand Down Expand Up @@ -67,13 +68,45 @@ var _ = Describe("DependencyWatchdog", func() {
Internal: &scalerapi.ProbeDetails{KubeconfigSecretName: "dependency-watchdog-internal-probe"},
PeriodSeconds: pointer.Int32(30),
},
DependantScales: []*scalerapi.DependantScaleDetails{{
ScaleRef: autoscalingv1.CrossVersionObjectReference{
APIVersion: "apps/v1",
Kind: "Deployment",
Name: "kube-controller-manager",
DependantScales: []*scalerapi.DependantScaleDetails{
{
ScaleRef: autoscalingv1.CrossVersionObjectReference{
APIVersion: "apps/v1",
Kind: "Deployment",
Name: "kube-controller-manager",
},
ScaleUpDelaySeconds: pointer.Int32(120),
},
{
ScaleRef: autoscalingv1.CrossVersionObjectReference{
APIVersion: "apps/v1",
Kind: "Deployment",
Name: "machine-controller-manager",
},
ScaleUpDelaySeconds: pointer.Int32(60),
ScaleRefDependsOn: []autoscalingv1.CrossVersionObjectReference{
{
APIVersion: appsv1.SchemeGroupVersion.String(),
rfranzke marked this conversation as resolved.
Show resolved Hide resolved
Kind: "Deployment",
Name: "kube-controller-manager",
},
},
},
}},
{
ScaleRef: autoscalingv1.CrossVersionObjectReference{
APIVersion: "apps/v1",
Kind: "Deployment",
Name: "cluster-autoscaler",
},
ScaleRefDependsOn: []autoscalingv1.CrossVersionObjectReference{
{
APIVersion: appsv1.SchemeGroupVersion.String(),
rfranzke marked this conversation as resolved.
Show resolved Hide resolved
Kind: "Deployment",
Name: "machine-controller-manager",
},
},
},
},
}))
Expect(err).NotTo(HaveOccurred())
})
Expand Down
2 changes: 1 addition & 1 deletion vendor/github.com/gardener/dependency-watchdog/pkg/restarter/api/types.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

7 changes: 5 additions & 2 deletions vendor/github.com/gardener/dependency-watchdog/pkg/scaler/api/types.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

7 changes: 5 additions & 2 deletions vendor/golang.org/x/mod/modfile/read.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.