fix: metrics-server upgrade from v1.15 to v1.16 (Azure#3691)

pkg/armhelpers/azurestack/kubeclient.go

@@ -11,6 +11,7 @@ import (
 	appsv1 "k8s.io/api/apps/v1"
 	v1 "k8s.io/api/core/v1"
 	policy "k8s.io/api/policy/v1beta1"
+	rbacv1 "k8s.io/api/rbac/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/fields"
@@ -117,11 +118,21 @@ func (c *KubernetesClientSetClient) SupportEviction() (string, error) {
 	return "", nil
 }
 
+// DeleteClusterRole deletes the passed in cluster role.
+func (c *KubernetesClientSetClient) DeleteClusterRole(role *rbacv1.ClusterRole) error {
+	return c.clientset.RbacV1().ClusterRoles().Delete(role.Name, &metav1.DeleteOptions{})
+}
+
 // DeleteDaemonSet deletes the passed in daemonset.
 func (c *KubernetesClientSetClient) DeleteDaemonSet(daemonset *appsv1.DaemonSet) error {
 	return c.clientset.AppsV1().DaemonSets(daemonset.Namespace).Delete(daemonset.Name, &metav1.DeleteOptions{})
 }
 
+// DeleteDeployment deletes the passed in daemonset.
+func (c *KubernetesClientSetClient) DeleteDeployment(deployment *appsv1.Deployment) error {
+	return c.clientset.AppsV1().Deployments(deployment.Namespace).Delete(deployment.Name, &metav1.DeleteOptions{})
+}
+
 // DeletePod deletes the passed in pod.
 func (c *KubernetesClientSetClient) DeletePod(pod *v1.Pod) error {
 	return c.clientset.CoreV1().Pods(pod.Namespace).Delete(pod.Name, &metav1.DeleteOptions{})

pkg/armhelpers/interfaces.go

@@ -19,6 +19,7 @@ import (
 	log "github.com/sirupsen/logrus"
 	appsv1 "k8s.io/api/apps/v1"
 	v1 "k8s.io/api/core/v1"
+	rbacv1 "k8s.io/api/rbac/v1"
 )
 
 // ResourceSkusResultPage
@@ -243,6 +244,8 @@ type KubernetesClient interface {
 	ListServiceAccounts(namespace string) (*v1.ServiceAccountList, error)
 	// GetDaemonSet returns details about DaemonSet with passed in name.
 	GetDaemonSet(namespace, name string) (*appsv1.DaemonSet, error)
+	// GetDeployment returns a given deployment in a namespace.
+	GetDeployment(namespace, name string) (*appsv1.Deployment, error)
 	// GetNode returns details about node with passed in name.
 	GetNode(name string) (*v1.Node, error)
 	// UpdateNode updates the node in the api server with the passed in info.
@@ -251,8 +254,12 @@ type KubernetesClient interface {
 	DeleteNode(name string) error
 	// SupportEviction queries the api server to discover if it supports eviction, and returns supported type if it is supported.
 	SupportEviction() (string, error)
+	// DeleteClusterRole deletes the passed in ClusterRole.
+	DeleteClusterRole(role *rbacv1.ClusterRole) error
 	// DeleteDaemonSet deletes the passed in DaemonSet.
 	DeleteDaemonSet(ds *appsv1.DaemonSet) error
+	// DeleteDeployment deletes the passed in Deployment.
+	DeleteDeployment(ds *appsv1.Deployment) error
 	// DeletePod deletes the passed in pod.
 	DeletePod(pod *v1.Pod) error
 	// DeleteServiceAccount deletes the passed in service account.
@@ -261,8 +268,6 @@ type KubernetesClient interface {
 	EvictPod(pod *v1.Pod, policyGroupVersion string) error
 	// WaitForDelete waits until all pods are deleted. Returns all pods not deleted and an error on failure.
 	WaitForDelete(logger *log.Entry, pods []v1.Pod, usingEviction bool) ([]v1.Pod, error)
-	// GetDeployment returns a given deployment in a namespace.
-	GetDeployment(namespace, name string) (*appsv1.Deployment, error)
 	// UpdateDeployment updates a deployment to match the given specification.
 	UpdateDeployment(namespace string, deployment *appsv1.Deployment) (*appsv1.Deployment, error)
 }

pkg/armhelpers/kubeclient.go

@@ -10,6 +10,7 @@ import (
 	appsv1 "k8s.io/api/apps/v1"
 	v1 "k8s.io/api/core/v1"
 	policy "k8s.io/api/policy/v1beta1"
+	rbacv1 "k8s.io/api/rbac/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/fields"
@@ -116,11 +117,21 @@ func (c *KubernetesClientSetClient) SupportEviction() (string, error) {
 	return "", nil
 }
 
+// DeleteClusterRole deletes the passed in cluster role.
+func (c *KubernetesClientSetClient) DeleteClusterRole(role *rbacv1.ClusterRole) error {
+	return c.clientset.RbacV1().ClusterRoles().Delete(role.Name, &metav1.DeleteOptions{})
+}
+
 // DeleteDaemonSet deletes the passed in daemonset.
 func (c *KubernetesClientSetClient) DeleteDaemonSet(daemonset *appsv1.DaemonSet) error {
 	return c.clientset.AppsV1().DaemonSets(daemonset.Namespace).Delete(daemonset.Name, &metav1.DeleteOptions{})
 }
 
+// DeleteDeployment deletes the passed in daemonset.
+func (c *KubernetesClientSetClient) DeleteDeployment(deployment *appsv1.Deployment) error {
+	return c.clientset.AppsV1().Deployments(deployment.Namespace).Delete(deployment.Name, &metav1.DeleteOptions{})
+}
+
 // DeletePod deletes the passed in pod.
 func (c *KubernetesClientSetClient) DeletePod(pod *v1.Pod) error {
 	return c.clientset.CoreV1().Pods(pod.Namespace).Delete(pod.Name, &metav1.DeleteOptions{})

pkg/armhelpers/mockclients.go

@@ -26,6 +26,7 @@ import (
 	log "github.com/sirupsen/logrus"
 	appsv1 "k8s.io/api/apps/v1"
 	v1 "k8s.io/api/core/v1"
+	rbacv1 "k8s.io/api/rbac/v1"
 )
 
 const (
@@ -85,7 +86,9 @@ type MockKubernetesClient struct {
 	FailDeleteServiceAccount  bool
 	FailSupportEviction       bool
 	FailDeletePod             bool
+	FailDeleteClusterRole     bool
 	FailDeleteDaemonSet       bool
+	FailDeleteDeployment      bool
 	FailEvictPod              bool
 	FailWaitForDelete         bool
 	ShouldSupportEviction     bool
@@ -395,6 +398,14 @@ func (mkc *MockKubernetesClient) SupportEviction() (string, error) {
 	return "", nil
 }
 
+//DeleteDeployment deletes the passed in daemonset
+func (mkc *MockKubernetesClient) DeleteClusterRole(role *rbacv1.ClusterRole) error {
+	if mkc.FailDeleteClusterRole {
+		return errors.New("ClusterRole failed")
+	}
+	return nil
+}
+
 //DeleteDaemonSet deletes the passed in daemonset
 func (mkc *MockKubernetesClient) DeleteDaemonSet(pod *appsv1.DaemonSet) error {
 	if mkc.FailDeleteDaemonSet {
@@ -403,6 +414,14 @@ func (mkc *MockKubernetesClient) DeleteDaemonSet(pod *appsv1.DaemonSet) error {
 	return nil
 }
 
+//DeleteDeployment deletes the passed in daemonset
+func (mkc *MockKubernetesClient) DeleteDeployment(pod *appsv1.Deployment) error {
+	if mkc.FailDeleteDeployment {
+		return errors.New("Deployment failed")
+	}
+	return nil
+}
+
 //DeletePod deletes the passed in pod
 func (mkc *MockKubernetesClient) DeletePod(pod *v1.Pod) error {
 	if mkc.FailDeletePod {

pkg/operations/kubernetesupgrade/upgrader.go

@@ -24,6 +24,7 @@ import (
 	"github.com/sirupsen/logrus"
 	appsv1 "k8s.io/api/apps/v1"
 	v1 "k8s.io/api/core/v1"
+	rbacv1 "k8s.io/api/rbac/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
@@ -97,29 +98,54 @@ func (ku *Upgrader) RunUpgrade() error {
 // handleUnreconcilableAddons ensures addon upgrades that addon-manager cannot handle by itself.
 // This method fails silently otherwide it would break test "Should not fail if a Kubernetes client cannot be created" (upgradecluster_test.go)
 func (ku *Upgrader) handleUnreconcilableAddons() {
+	upgradeVersion := ku.DataModel.Properties.OrchestratorProfile.OrchestratorVersion
 	// kube-proxy upgrade fails from v1.15 to 1.16: https://github.com/Azure/aks-engine/issues/3557
 	// deleting daemonset so addon-manager recreates instead of patching
-	upgradeVersion := ku.DataModel.Properties.OrchestratorProfile.OrchestratorVersion
 	if !common.IsKubernetesVersionGe(ku.CurrentVersion, "1.16.0") && common.IsKubernetesVersionGe(upgradeVersion, "1.16.0") {
 		ku.logger.Infof("Attempting to delete kube-proxy daemonset.")
 		client, err := ku.getKubernetesClient(getResourceTimeout)
 		if err != nil {
 			ku.logger.Errorf("Error getting Kubernetes client: %v", err)
 			return
 		}
-		ds := &appsv1.DaemonSet{
+		err = client.DeleteDaemonSet(&appsv1.DaemonSet{
 			ObjectMeta: metav1.ObjectMeta{
 				Namespace: "kube-system",
 				Name:      common.KubeProxyAddonName,
 			},
-		}
-		err = client.DeleteDaemonSet(ds)
+		})
 		if err != nil {
 			ku.logger.Errorf("Error deleting kube-proxy daemonset: %v", err)
-			return
 		}
 		ku.logger.Infof("Deleted kube-proxy daemonset. Addon-manager will recreate it.")
 	}
+	// metrics-server upgrade fails from v1.15 to 1.16 as the addon mode is EnsureExists for pre-v1.16 cluster
+	if !common.IsKubernetesVersionGe(ku.CurrentVersion, "1.16.0") && common.IsKubernetesVersionGe(upgradeVersion, "1.16.0") {
+		ku.logger.Infof("Attempting to delete metrics-server deployment.")
+		client, err := ku.getKubernetesClient(getResourceTimeout)
+		if err != nil {
+			ku.logger.Errorf("Error getting Kubernetes client: %v", err)
+			return
+		}
+		err = client.DeleteClusterRole(&rbacv1.ClusterRole{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: "system:metrics-server",
+			},
+		})
+		if err != nil {
+			ku.logger.Errorf("Error deleting metrics-server cluster role: %v", err)
+		}
+		err = client.DeleteDeployment(&appsv1.Deployment{
+			ObjectMeta: metav1.ObjectMeta{
+				Namespace: "kube-system",
+				Name:      common.MetricsServerAddonName,
+			},
+		})
+		if err != nil {
+			ku.logger.Errorf("Error deleting metrics-server deployment: %v", err)
+		}
+		ku.logger.Infof("Deleted metrics-server deployment. Addon-manager will recreate it.")
+	}
 }
 
 // Validate will run validation post upgrade