feat(stop_pods): neutralize the HPA as HPAScaleToZero may be in use

CHANGELOG.md

@@ -1,5 +1,10 @@
 # wiremind-kubernetes
 
+## v7.0.0 (2022-09-27)
+### BREAKING CHANGE
+- stop_pods: neutralize the HPA as `HPAScaleToZero` may be in use (HPA may scale up the Deployment even if replicas=0), a more straightforward solution will
+be available in the future see [here](https://github.com/kubernetes/enhancements/pull/2022). Of course `start_pods` repairs it. (encourage users to run this command to re-scale up).
+
 ## v6.4.0 (2022-04-13)
 ### Feat
 - kubernetes: add support for RbacAuthorizationV1.

VERSION

@@ -1 +1 @@
-6.4.0
+7.0.0

requirements.txt

@@ -2,23 +2,23 @@
 # This file is autogenerated by pip-compile with python 3.8
 # To update, run:
 #
-#    pip-compile --no-emit-index-url
+#    pip-compile --no-emit-index-url setup.py
 #
-cachetools==4.2.4
+cachetools==5.2.0
     # via google-auth
-certifi==2021.5.30
+certifi==2022.9.24
     # via
     #   kubernetes
     #   requests
-charset-normalizer==2.0.6
+charset-normalizer==2.1.1
     # via requests
-google-auth==2.2.1
+google-auth==2.11.1
     # via kubernetes
-idna==3.2
+idna==3.4
     # via requests
-kubernetes==18.20.0
+kubernetes==24.2.0
     # via wiremind-kubernetes (setup.py)
-oauthlib==3.1.1
+oauthlib==3.2.1
     # via requests-oauthlib
 pyasn1==0.4.8
     # via
@@ -28,25 +28,26 @@ pyasn1-modules==0.2.8
     # via google-auth
 python-dateutil==2.8.2
     # via kubernetes
-pyyaml==5.4.1
+pyyaml==6.0
     # via kubernetes
-requests==2.26.0
+requests==2.28.1
     # via
     #   kubernetes
     #   requests-oauthlib
-requests-oauthlib==1.3.0
+requests-oauthlib==1.3.1
     # via kubernetes
-rsa==4.7.2
+rsa==4.9
     # via google-auth
 six==1.16.0
     # via
+    #   google-auth
     #   kubernetes
     #   python-dateutil
-urllib3==1.26.7
+urllib3==1.26.12
     # via
     #   kubernetes
     #   requests
-websocket-client==1.2.1
+websocket-client==1.4.1
     # via kubernetes
 
 # The following packages are considered to be unsafe in a requirements file:

src/wiremind_kubernetes/kubernetes_client_additional_arguments.py

@@ -1,6 +1,7 @@
-import kubernetes.client
 from typing import Any, Dict
 
+import kubernetes.client
+
 
 class ClientWithArguments:
     """
@@ -55,6 +56,11 @@ def __init__(self, *args, dry_run: bool = False, **kwargs):
         super().__init__(client=kubernetes.client.BatchV1Api, dry_run=dry_run)
 
 
+class AutoscalingV1ApiWithArguments(ClientWithArguments):
+    def __init__(self, *args, dry_run: bool = False, **kwargs):
+        super().__init__(client=kubernetes.client.AutoscalingV1Api, dry_run=dry_run)
+
+
 class CustomObjectsApiWithArguments(ClientWithArguments):
     def __init__(self, *args, dry_run: bool = False, **kwargs):
         super().__init__(client=kubernetes.client.CustomObjectsApi, dry_run=dry_run)

src/wiremind_kubernetes/kubernetes_helper.py

@@ -1,7 +1,7 @@
 import logging
 import pprint
 import time
-from typing import Any, Dict, List, Optional, Union
+from typing import Any, Dict, List, Optional, Union, Generator
 
 import kubernetes
 
@@ -10,6 +10,7 @@
 from .kube_config import load_kubernetes_config
 from .kubernetes_client_additional_arguments import (
     AppV1ApiWithArguments,
+    AutoscalingV1ApiWithArguments,
     BatchV1ApiWithArguments,
     CoreV1ApiWithArguments,
     CustomObjectsApiWithArguments,
@@ -19,6 +20,8 @@
 
 logger = logging.getLogger(__name__)
 
+HPA_ID_PREFIX = "wm--disabled--kube"
+
 
 class KubernetesHelper:
     """
@@ -50,6 +53,9 @@ def __init__(
         self.client_corev1_api: kubernetes.client.CoreV1Api = CoreV1ApiWithArguments(dry_run=dry_run)
         self.client_appsv1_api: kubernetes.client.AppsV1Api = AppV1ApiWithArguments(dry_run=dry_run)
         self.client_batchv1_api: kubernetes.client.BatchV1Api = BatchV1ApiWithArguments(dry_run=dry_run)
+        self.client_autoscalingv1_api: kubernetes.client.AutoscalingV1Api = AutoscalingV1ApiWithArguments(
+            dry_run=dry_run
+        )
         self.client_custom_objects_api: kubernetes.client.CustomObjectsApi = CustomObjectsApiWithArguments(
             dry_run=dry_run
         )
@@ -214,6 +220,16 @@ def getPodNameFromDeployment(self, deployment_name, namespace_name):
             raise PodNotFound("No matching pod was found in the namespace %s" % (namespace_name))
         return pod_list[0].metadata.name
 
+    def get_deployment_hpa(self, *, deployment_name: str) -> Generator:
+        for hpa in self.client_autoscalingv1_api.list_namespaced_horizontal_pod_autoscaler(self.namespace).items:
+            if hpa.spec.scale_target_ref.kind == "Deployment" and hpa.spec.scale_target_ref.name == deployment_name:
+                yield hpa
+
+    def patch_deployment_hpa(self, *, hpa_name: str, body: Any):
+        self.client_autoscalingv1_api.patch_namespaced_horizontal_pod_autoscaler(
+            name=hpa_name, namespace=self.namespace, body=body
+        )
+
 
 class KubernetesDeploymentManager(NamespacedKubernetesHelper):
     """
@@ -305,6 +321,7 @@ def start_pods(self):
             if len(priority_dict):
                 scaled = True
                 for (name, expected_scale) in priority_dict.items():
+                    self.re_enable_hpa(deployment_name=name)
                     self.scale_up_deployment(name, expected_scale)
         if scaled:
             logger.info("Done scaling up application Deployments")
@@ -317,12 +334,26 @@ def _are_deployments_stopped(self, deployment_dict: Dict[str, int]) -> bool:
                 return False
         return True
 
+    @retry_kubernetes_request
+    def disable_hpa(self, *, deployment_name: str):
+        for hpa in self.get_deployment_hpa(deployment_name=deployment_name):
+            # Tell the hpa to manage a non-existing Deployment
+            hpa.spec.scale_target_ref.name = f"{HPA_ID_PREFIX}-{deployment_name}"
+            self.patch_deployment_hpa(hpa_name=hpa.metadata.name, body=hpa)
+
+    @retry_kubernetes_request
+    def re_enable_hpa(self, *, deployment_name: str):
+        for hpa in self.get_deployment_hpa(deployment_name=f"{HPA_ID_PREFIX}-{deployment_name}"):
+            hpa.spec.scale_target_ref.name = deployment_name
+            self.patch_deployment_hpa(hpa_name=hpa.metadata.name, body=hpa)
+
     def _stop_deployments(self, deployment_dict: Dict[str, int]):
         """
         Scale down a dict (deployment_name, expected_scale) of Deployments.
         """
         for _ in range(self.SCALE_DOWN_MAX_WAIT_TIME):
             for deployment_name in deployment_dict:
+                self.disable_hpa(deployment_name=deployment_name)
                 self.scale_down_deployment(deployment_name)
             if self._are_deployments_stopped(deployment_dict):
                 break

src/wiremind_kubernetes/tests/e2e_tests/helpers.py

@@ -1,7 +1,11 @@
+import json
 import logging
 import subprocess
 import sys
 import urllib
+from typing import Any, Dict
+
+from wiremind_kubernetes import run_command
 
 logger = logging.getLogger(__name__)
 
@@ -44,3 +48,10 @@ def get_k8s_username():
     )
     assert username
     return username
+
+
+def kubectl_get_json(*, resource: str, namespace: str, name: str) -> Dict[str, Any]:
+    output, *_ = run_command(
+        f"kubectl get {resource} {name} -n {namespace} --ignore-not-found -o json", return_result=True
+    )
+    return json.loads(output or "{}")

src/wiremind_kubernetes/tests/e2e_tests/manifests/3_hpa.yml

@@ -0,0 +1,27 @@
+apiVersion: autoscaling/v1
+kind: HorizontalPodAutoscaler
+metadata:
+  name: concerned
+spec:
+  maxReplicas: 10
+  minReplicas: 1
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: concerned
+  targetCPUUtilizationPercentage: 75
+
+---
+
+apiVersion: autoscaling/v1
+kind: HorizontalPodAutoscaler
+metadata:
+  name: unconcerned
+spec:
+  maxReplicas: 10
+  minReplicas: 1
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: unconcerned
+  targetCPUUtilizationPercentage: 75

src/wiremind_kubernetes/tests/e2e_tests/start_stop_test.py

@@ -3,6 +3,9 @@
 
 import wiremind_kubernetes
 from wiremind_kubernetes import KubernetesDeploymentManager
+from wiremind_kubernetes.kubernetes_helper import HPA_ID_PREFIX
+from wiremind_kubernetes.tests.e2e_tests.conftest import TEST_NAMESPACE
+from wiremind_kubernetes.tests.e2e_tests.helpers import kubectl_get_json
 
 logger = logging.getLogger(__name__)
 
@@ -11,6 +14,13 @@
 KubernetesDeploymentManager.SCALE_DOWN_MAX_WAIT_TIME = 30
 
 
+def assert_hpa_scale_target_ref_name(*, hpa_name, scale_target_ref_name: str):
+    assert (
+        kubectl_get_json(resource="hpa", namespace=TEST_NAMESPACE, name=hpa_name)["spec"]["scaleTargetRef"]["name"]
+        == scale_target_ref_name
+    )
+
+
 def are_deployments_ready(
     concerned_dm: KubernetesDeploymentManager, unconcerned_dm: KubernetesDeploymentManager
 ) -> bool:
@@ -52,6 +62,10 @@ def test_stop_start_all(concerned_dm, unconcerned_dm, populate_cluster, mocker):
     assert concerned_dm.is_deployment_stopped("concerned-high-priority")
     assert not unconcerned_dm.is_deployment_stopped("unconcerned")
 
+    # concerned HPA were disabled
+    assert_hpa_scale_target_ref_name(hpa_name="concerned", scale_target_ref_name=f"{HPA_ID_PREFIX}-concerned")
+    assert_hpa_scale_target_ref_name(hpa_name="unconcerned", scale_target_ref_name="unconcerned")
+
     # Test stop order to see if we honor priority (for in-depth testing of priority, see unit tests)
     scale_down_call_list = spied_scale_down_deployment.call_args_list
     assert scale_down_call_list[0][0][1] == "concerned-very-high-priority"
@@ -69,3 +83,7 @@ def test_stop_start_all(concerned_dm, unconcerned_dm, populate_cluster, mocker):
     assert not concerned_dm.is_deployment_stopped("concerned-high-priority")
     assert not concerned_dm.is_deployment_stopped("concerned-very-high-priority")
     assert not unconcerned_dm.is_deployment_stopped("unconcerned")
+
+    # concerned HPA were re-enabled
+    assert_hpa_scale_target_ref_name(hpa_name="concerned", scale_target_ref_name="concerned")
+    assert_hpa_scale_target_ref_name(hpa_name="unconcerned", scale_target_ref_name="unconcerned")