From e3af329e66cec19e382b5290de8a113c27a9dee4 Mon Sep 17 00:00:00 2001 From: derek10cloud Date: Tue, 8 Oct 2024 23:50:38 +0900 Subject: [PATCH 1/3] chore: remove unnecessary package --- stackql_deploy/cmd/base.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/stackql_deploy/cmd/base.py b/stackql_deploy/cmd/base.py index add2642..6929e97 100644 --- a/stackql_deploy/cmd/base.py +++ b/stackql_deploy/cmd/base.py @@ -1,6 +1,5 @@ -from ..lib.utils import perform_retries, run_stackql_command, catch_error_and_exit, run_stackql_query, export_vars, show_query, get_type, check_all_dicts -from ..lib.config import setup_environment, load_manifest, get_global_context_and_providers, get_full_context -from ..lib.templating import get_queries +from ..lib.utils import perform_retries, run_stackql_command, catch_error_and_exit, run_stackql_query, export_vars, show_query, check_all_dicts +from ..lib.config import setup_environment, load_manifest, get_global_context_and_providers class StackQLBase: def __init__(self, stackql, vars, logger, stack_dir, stack_env): From a21fbb6b0a317f0f101d8572f7b3c859ed169aae Mon Sep 17 00:00:00 2001 From: derek10cloud Date: Tue, 8 Oct 2024 23:51:54 +0900 Subject: [PATCH 2/3] feat: Support provider version pinning --- stackql_deploy/lib/utils.py | 107 ++++++++++++++++++++++++++++++++---- 1 file changed, 97 insertions(+), 10 deletions(-) diff --git a/stackql_deploy/lib/utils.py b/stackql_deploy/lib/utils.py index e7c743d..5c18a10 100644 --- a/stackql_deploy/lib/utils.py +++ b/stackql_deploy/lib/utils.py @@ -1,4 +1,8 @@ -import time, json, sys, subprocess +import time +import json +import sys +import subprocess +import re def catch_error_and_exit(errmsg, logger): logger.error(errmsg) @@ -81,6 +85,17 @@ def run_stackql_command(command, stackql, logger, ignore_errors=False, retries=0 while attempt <= retries: try: logger.debug(f"(utils.run_stackql_command) executing stackql command (attempt {attempt + 1}):\n\n{command}\n") + # If qyery is start with 'REGISTRY PULL', check version + if command.startswith("REGISTRY PULL"): + match = re.match(r'(REGISTRY PULL \w+)(::v[\d\.]+)?', command) + if match: + service_provider = match.group(1) + version = match.group(2) + if version: + command = f"{service_provider} {version[2:]}" + else: + raise ValueError("REGISTRY PULL command must be in the format 'REGISTRY PULL ::v' or 'REGISTRY PULL '") + result = stackql.executeStmt(command) logger.debug(f"(utils.run_stackql_command) stackql command result:\n\n{result}, type: {type(result)}\n") @@ -115,17 +130,89 @@ def run_stackql_command(command, stackql, logger, ignore_errors=False, retries=0 def pull_providers(providers, stackql, logger): logger.debug(f"(utils.pull_providers) stackql run time info:\n\n{json.dumps(stackql.properties(), indent=2)}\n") installed_providers = run_stackql_query("SHOW PROVIDERS", stackql, False, logger) # not expecting an error here - if len(installed_providers) == 0: - installed_names = set() - else: - installed_names = {provider["name"] for provider in installed_providers} + # check if the provider is already installed for provider in providers: - if provider not in installed_names: - logger.info(f"pulling provider '{provider}'...") - msg = run_stackql_command(f"REGISTRY PULL {provider}", stackql, logger) - logger.info(msg) + # check if the provider is a specific version + if "::" in provider: + name, version = provider.split("::") + check_provider_version_available(name, version, stackql, logger) + found = False + # provider is a version which will be installed + # installed is a version which is already installed + for installed in installed_providers: + # if name and version are the same, it's already installed + if installed["name"] == name and installed["version"] == version: + logger.info(f"provider '{provider}' is already installed.") + found = True + break + # if name is the same but the installed version is higher, + # it's already installed(latest version) + elif installed["name"] == name and is_installed_version_higher(installed["version"], version): + logger.warning(f"provider '{name}' version '{version}' is not available in the registry, but a higher version '{installed['version']}' is already installed.") + logger.warning("If you want to install the lower version, you must delete the higher version folder from the stackql providers directory.") + logger.info(f"provider {name}::{version} is already installed.") + found = True + break + # if not found, pull the provider + if not found: + logger.info(f"pulling provider '{provider}'...") + msg = run_stackql_command(f"REGISTRY PULL {provider}", stackql, logger) + logger.info(msg) + else: + found = False + # provider is a name which will be installed + # installed is a list of providers which are already installed + for installed in installed_providers: + if installed["name"] == provider: + logger.info(f"provider '{provider}' is already installed.") + found = True + break + # if not found, pull the provider + if not found: + logger.info(f"pulling provider '{provider}'...") + msg = run_stackql_command(f"REGISTRY PULL {provider}", stackql, logger) + logger.info(msg) + +def check_provider_version_available(provider_name, version, stackql, logger): + """Check if the provider version is available in the registry. + + Args: + provider_name (str): The name of the provider. + version (str): The version of the provider. + stackql (StackQL): The StackQL object. + logger (Logger): The logger object. + """ + query = f"REGISTRY LIST {provider_name}" + try: + result = run_stackql_query(query, stackql, True, logger) + # result[0]['versions'] is a string, not a list + # so we need to split it into a list + versions = result[0]['versions'].split(", ") + if version not in versions: + catch_error_and_exit(f"(utils.check_provider_version_available) version '{version}' not found for provider '{provider_name}', available versions: {versions}", logger) + except Exception: + catch_error_and_exit(f"(utils.check_provider_version_available) provider '{provider_name}' not found in registry", logger) + +def is_installed_version_higher(installed_version, requested_version): + """Check if the installed version is higher than the requested version. + + Args: + installed_version (str): v24.09.00251 + requested_version (str): v23.01.00104 + + Returns: + bool: True if installed version is higher than requested version, False otherwise + """ + + try: + int_installed = int(installed_version.replace("v", "").replace(".", "")) + int_requested = int(requested_version.replace("v", "").replace(".", "")) + if int_installed > int_requested: + return True else: - logger.info(f"provider '{provider}' is already installed.") + return False + except Exception: + catch_error_and_exit(f"(utils.is_installed_version_higher) version comparison failed: installed version '{installed_version}', requested version '{requested_version}'", logger) def run_test(resource, rendered_test_iql, stackql, logger, delete_test=False): try: From c558d7a697772e94f0714795a04c1204b7d8bc3d Mon Sep 17 00:00:00 2001 From: derek10cloud Date: Tue, 8 Oct 2024 23:54:14 +0900 Subject: [PATCH 3/3] Upgrade stackql-deploy version --- CHANGELOG.md | 4 ++++ setup.py | 2 +- stackql_deploy/__init__.py | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 19160a6..52776f1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## 1.7.7 (2024-10-09) + +- Supported version pinning for providers(aws, gcp, azure and etc) in `manifest` file + ## 1.7.6 (2024-10-07) - Added support for named `exports` (assigning an alias to the column name in the resource query file) - allows for more generalization and reuse of query files diff --git a/setup.py b/setup.py index eb9fed9..2a2df88 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ setup( name='stackql-deploy', - version='1.7.6', + version='1.7.7', description='Model driven resource provisioning and deployment framework using StackQL.', long_description=readme, long_description_content_type='text/x-rst', diff --git a/stackql_deploy/__init__.py b/stackql_deploy/__init__.py index 8bea1ec..ed2901d 100644 --- a/stackql_deploy/__init__.py +++ b/stackql_deploy/__init__.py @@ -1 +1 @@ -__version__ = '1.7.6' +__version__ = '1.7.7'