From bea6ec44b70fb65fd1d16628571633c9a586d955 Mon Sep 17 00:00:00 2001 From: "Farley Farley (yes, really)" Date: Tue, 25 Oct 2022 18:11:49 +1300 Subject: [PATCH] Handling low max disk size edge-case better, improving human-readable output (#4) --- helpers.py | 31 ++++++++++++++++++++++++++++--- main.py | 26 ++++++++++++++++++++++---- 2 files changed, 50 insertions(+), 7 deletions(-) diff --git a/helpers.py b/helpers.py index fc838f5..53a3987 100644 --- a/helpers.py +++ b/helpers.py @@ -88,9 +88,9 @@ def exit_gracefully(self, *args): ############################# # Simple header printing before the program starts, prints the variables this is configured for at runtime def printHeaderAndConfiguration(): - print("---------------------------------------------------------------") + print("-------------------------------------------------------------------------------------------------------------") print(" Volume Autoscaler - Configuration ") - print("---------------------------------------------------------------") + print("-------------------------------------------------------------------------------------------------------------") print(" Prometheus URL: {}".format(PROMETHEUS_URL)) print(" Prometheus Version: {}{}".format(PROMETHEUS_VERSION," (upgrade to >= 2.30.0 to prevent some false positives)" if version.parse(PROMETHEUS_VERSION) < version.parse("2.30.0") else "")) print(" Prometheus Labels: {{{}}}".format(PROMETHEUS_LABEL_MATCH)) @@ -105,7 +105,7 @@ def printHeaderAndConfiguration(): print(" Verbose Mode: is {}".format("ENABLED" if VERBOSE else "Disabled")) print(" Dry Run: is {}".format("ENABLED, no scaling will occur!" if DRY_RUN else "Disabled")) print(" HTTP Timeouts for k8s/prom: is {} seconds".format(HTTP_TIMEOUT)) - print("---------------------------------------------------------------") + print("-------------------------------------------------------------------------------------------------------------") # Figure out how many bytes to scale to based on the original size, scale up percent, minimum increment and maximum size @@ -137,6 +137,14 @@ def calculateBytesToScaleTo(original_size, scale_up_percent, min_increment, max_ print(e) return False +# Check if is integer or float +def is_integer_or_float(n): + try: + float(n) + except ValueError: + return False + else: + return float(n).is_integer() # Convert the K8s storage size definitions (eg: 10G, 5Ti, etc) into number of bytes def convert_storage_to_bytes(storage): @@ -205,6 +213,9 @@ def convert_bytes_to_storage(bytes): # Todo: Add Petabytes/Exobytes? + # Ensure its an intger + bytes = int(bytes) + # First, we'll try all base10 values... # Check if we can convert this into terrabytes result = try_numeric_format(bytes, 1000000000000, 'T') @@ -444,3 +455,17 @@ def send_kubernetes_event(namespace, name, reason, message, type="Normal"): print("Exception when calling CoreV1Api->create_namespaced_event: %s\n" % e) except: traceback.print_exc() + +# Print a sexy human readable dict for volume +def print_human_readable_volume_dict(input_dict): + for key in input_dict: + print(" {}: {}".format(key.rjust(24), input_dict[key]), end='') + if key in ['volume_size_spec','volume_size_spec_bytes','volume_size_status','volume_size_status_bytes','scale_up_min_increment','scale_up_max_increment','scale_up_max_size'] and is_integer_or_float(input_dict[key]): + print(" ({})".format(convert_bytes_to_storage(input_dict[key])), end='') + if key in ['scale_cooldown_time']: + print(" ({})".format(time.strftime('%H:%M:%S', time.gmtime(input_dict[key]))), end='') + if key in ['last_resized_at']: + print(" ({})".format(time.strftime('%Y-%m-%d %H:%M:%S %Z %z', time.localtime(input_dict[key]))), end='') + if key in ['scale_up_percent','scale_above_percent']: + print("%", end='') + print("") # Newline diff --git a/main.py b/main.py index 79c4d1e..7b54002 100755 --- a/main.py +++ b/main.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 import os import time -from helpers import INTERVAL_TIME, PROMETHEUS_URL, DRY_RUN, VERBOSE, get_settings_for_prometheus_metrics +from helpers import INTERVAL_TIME, PROMETHEUS_URL, DRY_RUN, VERBOSE, get_settings_for_prometheus_metrics, is_integer_or_float, print_human_readable_volume_dict from helpers import convert_bytes_to_storage, scale_up_pvc, testIfPrometheusIsAccessible, describe_all_pvcs, send_kubernetes_event from helpers import fetch_pvcs_from_prometheus, printHeaderAndConfiguration, calculateBytesToScaleTo, GracefulKiller from prometheus_client import start_http_server, Summary, Gauge, Counter, Info @@ -97,8 +97,9 @@ if VERBOSE: print("Volume {} is {}% in-use of the {} available".format(volume_description,volume_used_percent,pvcs_in_kubernetes[volume_description]['volume_size_status'])) print(" VERBOSE DETAILS:") - for key in pvcs_in_kubernetes[volume_description]: - print(" {}: {}".format(key, pvcs_in_kubernetes[volume_description][key])) + print("-------------------------------------------------------------------------------------------------------------") + print_human_readable_volume_dict(pvcs_in_kubernetes[volume_description]) + print("-------------------------------------------------------------------------------------------------------------") # Check if we are NOT in an alert condition if volume_used_percent < pvcs_in_kubernetes[volume_description]['scale_above_percent']: @@ -152,8 +153,25 @@ # If our resize bytes failed for some reason, eg putting invalid data into the annotations on the PV if resize_to_bytes == False: - print(" Error/Exception while trying to determine what to resize to, values causing failure:") + print("-------------------------------------------------------------------------------------------------------------") + print(" Error/Exception while trying to determine what to resize to, volume causing failure:") + print("-------------------------------------------------------------------------------------------------------------") print(pvcs_in_kubernetes[volume_description]) + print("-------------------------------------------------------------------------------------------------------------") + continue + + # If our resize bytes is less than our original size (because the user set the max-bytes to something too low) + if resize_to_bytes < pvcs_in_kubernetes[volume_description]['volume_size_status_bytes']: + print("-------------------------------------------------------------------------------------------------------------") + print(" Error/Exception while trying to scale this up. Is it possible your maximum SCALE_UP_MAX_SIZE is too small?") + print("-------------------------------------------------------------------------------------------------------------") + print(" Maximum Size: {} ({})".format(pvcs_in_kubernetes[volume_description]['scale_up_max_size'], convert_bytes_to_storage(pvcs_in_kubernetes[volume_description]['scale_up_max_size']))) + print(" Original Size: {} ({})".format(pvcs_in_kubernetes[volume_description]['volume_size_status_bytes'], convert_bytes_to_storage(pvcs_in_kubernetes[volume_description]['volume_size_status_bytes']))) + print(" Resize To: {} ({})".format(resize_to_bytes, convert_bytes_to_storage(resize_to_bytes))) + print("-------------------------------------------------------------------------------------------------------------") + print(" Volume causing failure:") + print_human_readable_volume_dict(pvcs_in_kubernetes[volume_description]) + print("-------------------------------------------------------------------------------------------------------------") continue # Check if we are already at the max volume size (either globally, or this-volume specific)