diff --git a/CHANGELOG.md b/CHANGELOG.md index 3f1d687..e57d3b7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # wiremind-kubernetes +## v7.0.0 (2022-09-27) +### BREAKING CHANGE +- stop_pods: neutralize the HPA as `HPAScaleToZero` may be in use (HPA may scale up the Deployment even if replicas=0), a more straightforward solution will +be available in the future see [here](https://github.com/kubernetes/enhancements/pull/2022). Of course `start_pods` repairs it. (encourage users to run this command to re-scale up). + ## v6.4.0 (2022-04-13) ### Feat - kubernetes: add support for RbacAuthorizationV1. diff --git a/VERSION b/VERSION index 19b860c..4122521 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -6.4.0 +7.0.0 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 7253409..9a5531a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,23 +2,23 @@ # This file is autogenerated by pip-compile with python 3.8 # To update, run: # -# pip-compile --no-emit-index-url +# pip-compile --no-emit-index-url setup.py # -cachetools==4.2.4 +cachetools==5.2.0 # via google-auth -certifi==2021.5.30 +certifi==2022.9.24 # via # kubernetes # requests -charset-normalizer==2.0.6 +charset-normalizer==2.1.1 # via requests -google-auth==2.2.1 +google-auth==2.11.1 # via kubernetes -idna==3.2 +idna==3.4 # via requests -kubernetes==18.20.0 +kubernetes==24.2.0 # via wiremind-kubernetes (setup.py) -oauthlib==3.1.1 +oauthlib==3.2.1 # via requests-oauthlib pyasn1==0.4.8 # via @@ -28,25 +28,26 @@ pyasn1-modules==0.2.8 # via google-auth python-dateutil==2.8.2 # via kubernetes -pyyaml==5.4.1 +pyyaml==6.0 # via kubernetes -requests==2.26.0 +requests==2.28.1 # via # kubernetes # requests-oauthlib -requests-oauthlib==1.3.0 +requests-oauthlib==1.3.1 # via kubernetes -rsa==4.7.2 +rsa==4.9 # via google-auth six==1.16.0 # via + # google-auth # kubernetes # python-dateutil -urllib3==1.26.7 +urllib3==1.26.12 # via # kubernetes # requests -websocket-client==1.2.1 +websocket-client==1.4.1 # via kubernetes # The following packages are considered to be unsafe in a requirements file: diff --git a/src/wiremind_kubernetes/kubernetes_client_additional_arguments.py b/src/wiremind_kubernetes/kubernetes_client_additional_arguments.py index 183d563..53734c3 100644 --- a/src/wiremind_kubernetes/kubernetes_client_additional_arguments.py +++ b/src/wiremind_kubernetes/kubernetes_client_additional_arguments.py @@ -1,6 +1,7 @@ -import kubernetes.client from typing import Any, Dict +import kubernetes.client + class ClientWithArguments: """ @@ -55,6 +56,11 @@ def __init__(self, *args, dry_run: bool = False, **kwargs): super().__init__(client=kubernetes.client.BatchV1Api, dry_run=dry_run) +class AutoscalingV1ApiWithArguments(ClientWithArguments): + def __init__(self, *args, dry_run: bool = False, **kwargs): + super().__init__(client=kubernetes.client.AutoscalingV1Api, dry_run=dry_run) + + class CustomObjectsApiWithArguments(ClientWithArguments): def __init__(self, *args, dry_run: bool = False, **kwargs): super().__init__(client=kubernetes.client.CustomObjectsApi, dry_run=dry_run) diff --git a/src/wiremind_kubernetes/kubernetes_helper.py b/src/wiremind_kubernetes/kubernetes_helper.py index 1ff23c8..c801054 100644 --- a/src/wiremind_kubernetes/kubernetes_helper.py +++ b/src/wiremind_kubernetes/kubernetes_helper.py @@ -1,7 +1,7 @@ import logging import pprint import time -from typing import Any, Dict, List, Optional, Union +from typing import Any, Dict, List, Optional, Union, Generator import kubernetes @@ -10,6 +10,7 @@ from .kube_config import load_kubernetes_config from .kubernetes_client_additional_arguments import ( AppV1ApiWithArguments, + AutoscalingV1ApiWithArguments, BatchV1ApiWithArguments, CoreV1ApiWithArguments, CustomObjectsApiWithArguments, @@ -19,6 +20,8 @@ logger = logging.getLogger(__name__) +HPA_ID_PREFIX = "wm--disabled--kube" + class KubernetesHelper: """ @@ -50,6 +53,9 @@ def __init__( self.client_corev1_api: kubernetes.client.CoreV1Api = CoreV1ApiWithArguments(dry_run=dry_run) self.client_appsv1_api: kubernetes.client.AppsV1Api = AppV1ApiWithArguments(dry_run=dry_run) self.client_batchv1_api: kubernetes.client.BatchV1Api = BatchV1ApiWithArguments(dry_run=dry_run) + self.client_autoscalingv1_api: kubernetes.client.AutoscalingV1Api = AutoscalingV1ApiWithArguments( + dry_run=dry_run + ) self.client_custom_objects_api: kubernetes.client.CustomObjectsApi = CustomObjectsApiWithArguments( dry_run=dry_run ) @@ -214,6 +220,16 @@ def getPodNameFromDeployment(self, deployment_name, namespace_name): raise PodNotFound("No matching pod was found in the namespace %s" % (namespace_name)) return pod_list[0].metadata.name + def get_deployment_hpa(self, *, deployment_name: str) -> Generator: + for hpa in self.client_autoscalingv1_api.list_namespaced_horizontal_pod_autoscaler(self.namespace).items: + if hpa.spec.scale_target_ref.kind == "Deployment" and hpa.spec.scale_target_ref.name == deployment_name: + yield hpa + + def patch_deployment_hpa(self, *, hpa_name: str, body: Any): + self.client_autoscalingv1_api.patch_namespaced_horizontal_pod_autoscaler( + name=hpa_name, namespace=self.namespace, body=body + ) + class KubernetesDeploymentManager(NamespacedKubernetesHelper): """ @@ -305,6 +321,7 @@ def start_pods(self): if len(priority_dict): scaled = True for (name, expected_scale) in priority_dict.items(): + self.re_enable_hpa(deployment_name=name) self.scale_up_deployment(name, expected_scale) if scaled: logger.info("Done scaling up application Deployments") @@ -317,12 +334,26 @@ def _are_deployments_stopped(self, deployment_dict: Dict[str, int]) -> bool: return False return True + @retry_kubernetes_request + def disable_hpa(self, *, deployment_name: str): + for hpa in self.get_deployment_hpa(deployment_name=deployment_name): + # Tell the hpa to manage a non-existing Deployment + hpa.spec.scale_target_ref.name = f"{HPA_ID_PREFIX}-{deployment_name}" + self.patch_deployment_hpa(hpa_name=hpa.metadata.name, body=hpa) + + @retry_kubernetes_request + def re_enable_hpa(self, *, deployment_name: str): + for hpa in self.get_deployment_hpa(deployment_name=f"{HPA_ID_PREFIX}-{deployment_name}"): + hpa.spec.scale_target_ref.name = deployment_name + self.patch_deployment_hpa(hpa_name=hpa.metadata.name, body=hpa) + def _stop_deployments(self, deployment_dict: Dict[str, int]): """ Scale down a dict (deployment_name, expected_scale) of Deployments. """ for _ in range(self.SCALE_DOWN_MAX_WAIT_TIME): for deployment_name in deployment_dict: + self.disable_hpa(deployment_name=deployment_name) self.scale_down_deployment(deployment_name) if self._are_deployments_stopped(deployment_dict): break diff --git a/src/wiremind_kubernetes/tests/e2e_tests/helpers.py b/src/wiremind_kubernetes/tests/e2e_tests/helpers.py index 2d37f83..e0176e8 100644 --- a/src/wiremind_kubernetes/tests/e2e_tests/helpers.py +++ b/src/wiremind_kubernetes/tests/e2e_tests/helpers.py @@ -1,7 +1,11 @@ +import json import logging import subprocess import sys import urllib +from typing import Any, Dict + +from wiremind_kubernetes import run_command logger = logging.getLogger(__name__) @@ -44,3 +48,10 @@ def get_k8s_username(): ) assert username return username + + +def kubectl_get_json(*, resource: str, namespace: str, name: str) -> Dict[str, Any]: + output, *_ = run_command( + f"kubectl get {resource} {name} -n {namespace} --ignore-not-found -o json", return_result=True + ) + return json.loads(output or "{}") diff --git a/src/wiremind_kubernetes/tests/e2e_tests/manifests/3_hpa.yml b/src/wiremind_kubernetes/tests/e2e_tests/manifests/3_hpa.yml new file mode 100644 index 0000000..73472df --- /dev/null +++ b/src/wiremind_kubernetes/tests/e2e_tests/manifests/3_hpa.yml @@ -0,0 +1,27 @@ +apiVersion: autoscaling/v1 +kind: HorizontalPodAutoscaler +metadata: + name: concerned +spec: + maxReplicas: 10 + minReplicas: 1 + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: concerned + targetCPUUtilizationPercentage: 75 + +--- + +apiVersion: autoscaling/v1 +kind: HorizontalPodAutoscaler +metadata: + name: unconcerned +spec: + maxReplicas: 10 + minReplicas: 1 + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: unconcerned + targetCPUUtilizationPercentage: 75 diff --git a/src/wiremind_kubernetes/tests/e2e_tests/start_stop_test.py b/src/wiremind_kubernetes/tests/e2e_tests/start_stop_test.py index 905d7f3..3aeaf84 100644 --- a/src/wiremind_kubernetes/tests/e2e_tests/start_stop_test.py +++ b/src/wiremind_kubernetes/tests/e2e_tests/start_stop_test.py @@ -3,6 +3,9 @@ import wiremind_kubernetes from wiremind_kubernetes import KubernetesDeploymentManager +from wiremind_kubernetes.kubernetes_helper import HPA_ID_PREFIX +from wiremind_kubernetes.tests.e2e_tests.conftest import TEST_NAMESPACE +from wiremind_kubernetes.tests.e2e_tests.helpers import kubectl_get_json logger = logging.getLogger(__name__) @@ -11,6 +14,13 @@ KubernetesDeploymentManager.SCALE_DOWN_MAX_WAIT_TIME = 30 +def assert_hpa_scale_target_ref_name(*, hpa_name, scale_target_ref_name: str): + assert ( + kubectl_get_json(resource="hpa", namespace=TEST_NAMESPACE, name=hpa_name)["spec"]["scaleTargetRef"]["name"] + == scale_target_ref_name + ) + + def are_deployments_ready( concerned_dm: KubernetesDeploymentManager, unconcerned_dm: KubernetesDeploymentManager ) -> bool: @@ -52,6 +62,10 @@ def test_stop_start_all(concerned_dm, unconcerned_dm, populate_cluster, mocker): assert concerned_dm.is_deployment_stopped("concerned-high-priority") assert not unconcerned_dm.is_deployment_stopped("unconcerned") + # concerned HPA were disabled + assert_hpa_scale_target_ref_name(hpa_name="concerned", scale_target_ref_name=f"{HPA_ID_PREFIX}-concerned") + assert_hpa_scale_target_ref_name(hpa_name="unconcerned", scale_target_ref_name="unconcerned") + # Test stop order to see if we honor priority (for in-depth testing of priority, see unit tests) scale_down_call_list = spied_scale_down_deployment.call_args_list assert scale_down_call_list[0][0][1] == "concerned-very-high-priority" @@ -69,3 +83,7 @@ def test_stop_start_all(concerned_dm, unconcerned_dm, populate_cluster, mocker): assert not concerned_dm.is_deployment_stopped("concerned-high-priority") assert not concerned_dm.is_deployment_stopped("concerned-very-high-priority") assert not unconcerned_dm.is_deployment_stopped("unconcerned") + + # concerned HPA were re-enabled + assert_hpa_scale_target_ref_name(hpa_name="concerned", scale_target_ref_name="concerned") + assert_hpa_scale_target_ref_name(hpa_name="unconcerned", scale_target_ref_name="unconcerned")