From aba3a230e3aaa10a800174c6e22b31fb0fa0c03a Mon Sep 17 00:00:00 2001 From: Koby Meir Date: Sun, 30 Jul 2023 23:55:27 +0300 Subject: [PATCH] Uninstall one-by-one with waiting for the status from the server not to be in "create / update / delete operation is already in progress (10102)" (#28586) --- .gitignore | 1 + .gitlab/ci/.gitlab-ci.bucket-upload.yml | 10 +- .../Marketplace/search_and_uninstall_pack.py | 151 ++++++++++++++++-- ...onfigure_and_test_integration_instances.py | 70 ++++---- .../uninstall_packs_and_reset_bucket_cloud.sh | 4 +- 5 files changed, 180 insertions(+), 56 deletions(-) diff --git a/.gitignore b/.gitignore index 6228594d099..e545e4b95de 100644 --- a/.gitignore +++ b/.gitignore @@ -93,6 +93,7 @@ test_runner.sh # log files demisto_sdk_debug.log demisto_sdk_debug.log.* +*.log # Ignore Modeling Rules test conf Packs/**/ModelingRules/**/**/*_testdata.json \ No newline at end of file diff --git a/.gitlab/ci/.gitlab-ci.bucket-upload.yml b/.gitlab/ci/.gitlab-ci.bucket-upload.yml index aec19eee75f..245e5c00b39 100644 --- a/.gitlab/ci/.gitlab-ci.bucket-upload.yml +++ b/.gitlab/ci/.gitlab-ci.bucket-upload.yml @@ -221,18 +221,18 @@ install-packs-in-server-master: - CLOUD_SERVERS_PATH=$(cat $CLOUD_SERVERS_FILE) - echo ${CLOUD_API_KEYS} > "cloud_api_keys.json" + - section_start "Clean Machine" + - ./Tests/scripts/uninstall_packs_and_reset_bucket_cloud.sh || EXIT_CODE=$? + - section_end "Clean Machine" + - | if [ "$INSTANCE_ROLE" == XSIAM ]; then section_start "Run end to end sanity tests" - + python3 -m pytest ./Tests/tests_end_to_end_xsiam -v --cloud_machine "$CLOUD_CHOSEN_MACHINE_ID" --cloud_servers_path "$CLOUD_SERVERS_PATH" --cloud_servers_api_keys "cloud_api_keys.json" --disable-warnings section_end "Run end to end sanity tests" fi - - section_start "Clean Machine" - - ./Tests/scripts/uninstall_packs_and_reset_bucket_cloud.sh - - section_end "Clean Machine" - - section_start "Get Instance Variables" - echo INSTANCE_ROLE="$INSTANCE_ROLE" - echo INSTANCE_CREATED="$INSTANCE_CREATED" diff --git a/Tests/Marketplace/search_and_uninstall_pack.py b/Tests/Marketplace/search_and_uninstall_pack.py index 5395a20cb45..db23a418d02 100644 --- a/Tests/Marketplace/search_and_uninstall_pack.py +++ b/Tests/Marketplace/search_and_uninstall_pack.py @@ -1,15 +1,21 @@ -import ast import argparse +import ast import math import os import sys +from datetime import datetime, timedelta +from time import sleep import demisto_client -from Tests.configure_and_test_integration_instances import CloudBuild +from demisto_client.demisto_api.rest import ApiException +from urllib3.exceptions import HTTPWarning, HTTPError + +from Tests.Marketplace.configure_and_install_packs import search_and_install_packs_and_their_dependencies +from Tests.configure_and_test_integration_instances import CloudBuild, get_custom_user_agent from Tests.scripts.utils import logging_wrapper as logging from Tests.scripts.utils.log_util import install_logging -from Tests.Marketplace.configure_and_install_packs import search_and_install_packs_and_their_dependencies -from time import sleep + +ALREADY_IN_PROGRESS = "create / update / delete operation is already in progress (10102)" def get_all_installed_packs(client: demisto_client, unremovable_packs: list): @@ -77,23 +83,140 @@ def uninstall_all_packs_one_by_one(client: demisto_client, hostname, unremovable return uninstalled_count == len(packs_to_uninstall) -def uninstall_pack(client: demisto_client, pack_id: str): +def get_updating_status(client: demisto_client, + attempts_count: int = 5, + sleep_interval: int = 60, + ) -> tuple[bool, bool | None]: + try: + for attempt in range(attempts_count - 1, -1, -1): + try: + logging.info(f"Getting installation/update status, Attempt: {attempts_count - attempt}/{attempts_count}") + response, status_code, headers = demisto_client.generic_request_func(client, + path='/content/updating', + method='GET', + accept='application/json', + _request_timeout=None) + + if 200 <= status_code < 300 and status_code != 204: + # the endpoint simply returns a string (rather than a json object.) + updating_status = 'true' in str(response).lower() + logging.info(f"Got updating status: {updating_status}") + return True, updating_status + else: + logging.info(f"Got bad response for updating status: {response}") + + if not attempt: + raise Exception(f"Got bad status code: {status_code}, headers: {headers}") + + logging.warning(f"Got bad status code: {status_code} from the server, headers: {headers}") + + except ApiException as ex: + if not attempt: # exhausted all attempts, understand what happened and exit. + # Unknown exception reason, re-raise. + raise Exception(f"Got status {ex.status} from server, message: {ex.body}, headers: {ex.headers}") from ex + except (HTTPError, HTTPWarning) as http_ex: + if not attempt: + raise Exception("Failed to perform http request to the server") from http_ex + + # There are more attempts available, sleep and retry. + logging.debug(f"Failed to get installation/update status, sleeping for {sleep_interval} seconds.") + sleep(sleep_interval) + + except Exception as e: + logging.exception(f'The request to get update status has failed. Additional info: {str(e)}') + return False, None + + +def wait_until_not_updating(client: demisto_client, + attempts_count: int = 2, + sleep_interval: int = 30, + maximum_time_to_wait: int = 600, + ) -> bool: """ Args: client (demisto_client): The client to connect to. - pack_id: packs id to uninstall + attempts_count (int): The number of attempts to install the packs. + sleep_interval (int): The sleep interval, in seconds, between install attempts. + maximum_time_to_wait (int): The maximum time to wait for the server to exit the updating mode, in seconds. + Returns: + Boolean - If the operation succeeded. + + """ + end_time = datetime.utcnow() + timedelta(seconds=maximum_time_to_wait) + while datetime.utcnow() <= end_time: + success, updating_status = get_updating_status(client) + if success: + if not updating_status: + return True + logging.debug(f"Server is still installation/updating status, sleeping for {sleep_interval} seconds.") + sleep(sleep_interval) + else: + if attempts_count := attempts_count - 1: + logging.debug(f"failed to get installation/updating status, sleeping for {sleep_interval} seconds.") + sleep(sleep_interval) + else: + logging.info("Exiting after exhausting all attempts") + return False + logging.info(f"Exiting after exhausting the allowed time:{maximum_time_to_wait} seconds") + return False + +def uninstall_pack(client: demisto_client, + pack_id: str, + attempts_count: int = 5, + sleep_interval: int = 60, + ): + """ + + Args: + client (demisto_client): The client to connect to. + pack_id: packs id to uninstall + attempts_count (int): The number of attempts to install the packs. + sleep_interval (int): The sleep interval, in seconds, between install attempts. Returns: Boolean - If the operation succeeded. """ try: - demisto_client.generic_request_func(client, - path=f'/contentpacks/installed/{pack_id}', - method='DELETE', - accept='application/json', - _request_timeout=None) + for attempt in range(attempts_count - 1, -1, -1): + try: + logging.info(f"Uninstalling packs {pack_id}, Attempt: {attempts_count - attempt}/{attempts_count}") + response, status_code, headers = demisto_client.generic_request_func(client, + path=f'/contentpacks/installed/{pack_id}', + method='DELETE', + accept='application/json', + _request_timeout=None) + + if 200 <= status_code < 300 and status_code != 204: + logging.success(f'Pack: {pack_id} was successfully uninstalled from the server') + break + + if not attempt: + raise Exception(f"Got bad status code: {status_code}, headers: {headers}") + + logging.warning(f"Got bad status code: {status_code} from the server, headers: {headers}") + + except ApiException as ex: + + if ALREADY_IN_PROGRESS in ex.body: + wait_succeeded = wait_until_not_updating(client) + if not wait_succeeded: + raise Exception( + "Failed to wait for the server to exit installation/updating status" + ) from ex + + if not attempt: # exhausted all attempts, understand what happened and exit. + # Unknown exception reason, re-raise. + raise Exception(f"Got {ex.status} from server, message: {ex.body}, headers: {ex.headers}") from ex + except (HTTPError, HTTPWarning) as http_ex: + if not attempt: + raise Exception("Failed to perform http request to the server") from http_ex + + # There are more attempts available, sleep and retry. + logging.debug(f"failed to uninstall pack: {pack_id}, sleeping for {sleep_interval} seconds.") + sleep(sleep_interval) + return True except Exception as e: logging.exception(f'The request to uninstall packs has failed. Additional info: {str(e)}') @@ -235,6 +358,7 @@ def options_handler(): parser.add_argument('--cloud_servers_api_keys', help='Path to the file with cloud Servers api keys.') parser.add_argument('--unremovable_packs', help='List of packs that cant be removed.') parser.add_argument('--one-by-one', help='Uninstall pack one pack at a time.', action='store_true') + parser.add_argument('--build-number', help='CI job number where the instances were created', required=True) options = parser.parse_args() @@ -244,7 +368,7 @@ def options_handler(): def main(): install_logging('cleanup_cloud_instance.log', logger=logging) - # in cloud we dont use demisto username + # In Cloud, We don't use demisto username os.environ.pop('DEMISTO_USERNAME', None) options = options_handler() @@ -259,6 +383,9 @@ def main(): verify_ssl=False, api_key=api_key, auth_id=xdr_auth_id) + client.api_client.user_agent = get_custom_user_agent(options.build_number) + logging.debug(f'Setting user agent on client to: {client.api_client.user_agent}') + # We are syncing marketplace since we are copying production bucket to build bucket and if packs were configured # in earlier builds they will appear in the bucket as it is cached. sync_marketplace(client=client) diff --git a/Tests/configure_and_test_integration_instances.py b/Tests/configure_and_test_integration_instances.py index c1964b572ae..cd49fb614cb 100644 --- a/Tests/configure_and_test_integration_instances.py +++ b/Tests/configure_and_test_integration_instances.py @@ -14,7 +14,6 @@ from pprint import pformat from threading import Thread from time import sleep -from typing import List, Tuple, Union, Set from urllib.parse import quote_plus import demisto_client @@ -85,6 +84,10 @@ class Running(IntEnum): WITH_LOCAL_SERVER = 2 +def get_custom_user_agent(build_number): + return f"demisto-py/dev (Build:{build_number})" + + class Server: def __init__(self): @@ -94,9 +97,6 @@ def __init__(self): self.name = '' self.build_number = 'unknown' - def get_custom_user_agent(self): - return f"demisto-py/dev (Build:{self.build_number})" - class CloudServer(Server): @@ -127,7 +127,7 @@ def reconnect_client(self): verify_ssl=False, api_key=self.api_key, auth_id=self.xdr_auth_id) - custom_user_agent = self.get_custom_user_agent() + custom_user_agent = get_custom_user_agent(self.build_number) logging.debug(f'Setting user agent on client to:{custom_user_agent}') self.__client.api_client.user_agent = custom_user_agent return self.__client @@ -158,7 +158,7 @@ def reconnect_client(self): verify_ssl=False, username=self.user_name, password=self.password) - custom_user_agent = self.get_custom_user_agent() + custom_user_agent = get_custom_user_agent(self.build_number) logging.debug(f'Setting user agent on client to:{custom_user_agent}') self.__client.api_client.user_agent = custom_user_agent return self.__client @@ -174,7 +174,7 @@ def exec_command(self, command): stderr=subprocess.STDOUT) -def get_id_set(id_set_path) -> Union[dict, None]: +def get_id_set(id_set_path) -> dict | None: """ Used to collect the ID set so it can be passed to the Build class on init. @@ -240,7 +240,7 @@ def fetch_tests_list(tests_to_run_path: str): :return: List of tests if there are any, otherwise empty list. """ tests_to_run = [] - with open(tests_to_run_path, "r") as filter_file: + with open(tests_to_run_path) as filter_file: tests_from_file = filter_file.readlines() for test_from_file in tests_from_file: test_clean = test_from_file.rstrip() @@ -256,7 +256,7 @@ def fetch_pack_ids_to_install(packs_to_install_path: str): :return: List of Pack IDs if there are any, otherwise empty list. """ tests_to_run = [] - with open(packs_to_install_path, "r") as filter_file: + with open(packs_to_install_path) as filter_file: tests_from_file = filter_file.readlines() for test_from_file in tests_from_file: test_clean = test_from_file.rstrip() @@ -303,7 +303,7 @@ def disable_instances(self): for server in self.servers: disable_all_integrations(server.client) - def get_changed_integrations(self, packs_not_to_install: Set[str] | None = None) -> Tuple[List[str], List[str]]: + def get_changed_integrations(self, packs_not_to_install: set[str] | None = None) -> tuple[list[str], list[str]]: """ Return 2 lists - list of new integrations names and list of modified integrations names since the commit of the git_sha1. The modified list is exclude the packs_not_to_install and the new list is including it @@ -359,7 +359,7 @@ def install_packs(self, pack_ids=None, install_packs_one_by_one=False): return installed_content_packs_successfully - def get_tests(self) -> List[dict]: + def get_tests(self) -> list[dict]: """ Selects the tests from that should be run in this execution and filters those that cannot run in this server version Args: @@ -461,7 +461,7 @@ def configure_modified_and_new_integrations(self, self: The build object modified_integrations_to_configure: Integrations to configure that are already exists new_integrations_to_configure: Integrations to configure that were created in this build - demisto_client: A demisto client + demisto_client_: A demisto client Returns: A tuple with two lists: @@ -486,7 +486,7 @@ def instance_testing(self, all_module_instances: list, pre_update: bool, use_mock: bool = True, - first_call: bool = True) -> Tuple[set, set]: + first_call: bool = True) -> tuple[set, set]: """ Runs 'test-module' command for the instances detailed in `all_module_instances` Args: @@ -599,19 +599,17 @@ def marketplace_name(self) -> str: def configure_servers_and_restart(self): manual_restart = Build.run_environment == Running.WITH_LOCAL_SERVER for server in self.servers: - configurations = dict() - configure_types = [] + configurations = {} if is_redhat_instance(server.internal_ip): configurations.update(DOCKER_HARDENING_CONFIGURATION_FOR_PODMAN) configurations.update(NO_PROXY_CONFIG) configurations['python.pass.extra.keys'] += "##--network=slirp4netns:cidr=192.168.0.0/16" else: configurations.update(DOCKER_HARDENING_CONFIGURATION) - configure_types.append('docker hardening') - configure_types.append('marketplace') + configure_types = ['docker hardening', 'marketplace'] configurations.update(MARKET_PLACE_CONFIGURATION) - error_msg = 'failed to set {} configurations'.format(' and '.join(configure_types)) + error_msg = f"failed to set {' and '.join(configure_types)} configurations" server.add_server_configuration(configurations, error_msg=error_msg, restart=not manual_restart) if manual_restart: @@ -688,7 +686,6 @@ def test_integration_with_mock(self, instance: dict, pre_update: bool): Runs 'test-module' for given integration with mitmproxy In case the playback mode fails and this is a pre-update run - a record attempt will be executed. Args: - build: An object containing the current build info. instance: A dict containing the instance details pre_update: Whether this instance testing is before or after the content update on the server. @@ -1371,7 +1368,7 @@ def group_integrations(integrations, skipped_integrations_conf, new_integrations integration_to_status = {} for integration in integrations: integration_name = integration.get('name', '') - if integration_name in skipped_integrations_conf.keys(): + if integration_name in skipped_integrations_conf: continue if integration_name in new_integrations_names: @@ -1436,7 +1433,7 @@ def update_content_on_demisto_instance(client, server, ami_name): # verify the asset id matches the circleci build number / asset_id in the content-descriptor.json release, asset_id = get_content_version_details(client, ami_name) logging.info(f'Content Release Version: {release}') - with open('./artifacts/content-descriptor.json', 'r') as cd_file: + with open('./artifacts/content-descriptor.json') as cd_file: cd_json = json.loads(cd_file.read()) cd_release = cd_json.get('release') cd_asset_id = cd_json.get('assetId') @@ -1569,7 +1566,7 @@ def get_servers(env_results, instance_role): def get_json_file(path): - with open(path, 'r') as json_file: + with open(path) as json_file: return json.loads(json_file.read()) @@ -1651,7 +1648,7 @@ def test_pack_zip(content_path, target, packs: list = None): if not test_path.endswith('.yml'): continue test = test.name - with open(test_path, 'r') as test_file: + with open(test_path) as test_file: if not (test.startswith(('playbook-', 'script-'))): test_type = find_type(_dict=yaml.safe_load(test_file), file_type='yml').value test_file.seek(0) @@ -1678,7 +1675,7 @@ def get_non_added_packs_ids(build: Build): added_files = added_files if not added_contrib_files else '\n'.join([added_files, added_contrib_files]) added_files = filter(lambda x: x, added_files.split('\n')) - added_pack_ids = map(lambda x: x.split('/')[1], added_files) + added_pack_ids = (x.split('/')[1] for x in added_files) # build.pack_ids_to_install contains new packs and modified. added_pack_ids contains new packs only. return set(build.pack_ids_to_install) - set(added_pack_ids) @@ -1692,7 +1689,9 @@ def run_git_diff(pack_name: str, build: Build) -> str: Returns: (str): The git diff output. """ - compare_against = 'origin/master{}'.format('' if not build.branch_name == 'master' else '~1') + compare_against = ( + f"origin/master{'' if build.branch_name != 'master' else '~1'}" + ) return run_command(f'git diff {compare_against}..{build.branch_name} -- Packs/{pack_name}/pack_metadata.json') @@ -1706,13 +1705,10 @@ def check_hidden_field_changed(pack_name: str, build: Build) -> bool: (bool): True if the pack transformed to non-hidden. """ diff = run_git_diff(pack_name, build) - for diff_line in diff.splitlines(): - if '"hidden": false' in diff_line and diff_line.split()[0].startswith('+'): - return True - return False + return any('"hidden": false' in diff_line and diff_line.split()[0].startswith('+') for diff_line in diff.splitlines()) -def get_turned_non_hidden_packs(modified_packs_names: Set[str], build: Build) -> Set[str]: +def get_turned_non_hidden_packs(modified_packs_names: set[str], build: Build) -> set[str]: """ Return a set of packs which turned from hidden to non-hidden. Args: @@ -1740,7 +1736,7 @@ def create_build_object() -> Build: raise Exception(f"Wrong Build object type {options.build_object_type}.") -def packs_names_to_integrations_names(turned_non_hidden_packs_names: Set[str]) -> List[str]: +def packs_names_to_integrations_names(turned_non_hidden_packs_names: set[str]) -> list[str]: """ Convert packs names to the integrations names contained in it. Args: @@ -1761,8 +1757,8 @@ def packs_names_to_integrations_names(turned_non_hidden_packs_names: Set[str]) - return hidden_integrations_names -def update_integration_lists(new_integrations_names: List[str], packs_not_to_install: Set[str] | None, - modified_integrations_names: List[str]) -> Tuple[List[str], List[str]]: +def update_integration_lists(new_integrations_names: list[str], packs_not_to_install: set[str] | None, + modified_integrations_names: list[str]) -> tuple[list[str], list[str]]: """ Add the turned non-hidden integrations names to the new integrations names list and remove it from modified integrations names. @@ -1785,7 +1781,7 @@ def update_integration_lists(new_integrations_names: List[str], packs_not_to_ins return list(set(new_integrations_names)), modified_integrations_names -def filter_new_to_marketplace_packs(build: Build, modified_pack_names: Set[str]) -> Set[str]: +def filter_new_to_marketplace_packs(build: Build, modified_pack_names: set[str]) -> set[str]: """ Return a set of packs that is new to the marketplace. Args: @@ -1802,7 +1798,7 @@ def filter_new_to_marketplace_packs(build: Build, modified_pack_names: Set[str]) return first_added_to_marketplace -def get_packs_to_install(build: Build) -> Tuple[Set[str], Set[str]]: +def get_packs_to_install(build: Build) -> tuple[set[str], set[str]]: """ Return a set of packs to install only in the pre-update, and set to install in post-update. Args: @@ -1831,8 +1827,8 @@ def get_packs_to_install(build: Build) -> Tuple[Set[str], Set[str]]: return packs_to_install_in_pre_update, non_hidden_packs -def get_packs_with_higher_min_version(packs_names: Set[str], - server_numeric_version: str) -> Set[str]: +def get_packs_with_higher_min_version(packs_names: set[str], + server_numeric_version: str) -> set[str]: """ Return a set of packs that have higher min version than the server version. diff --git a/Tests/scripts/uninstall_packs_and_reset_bucket_cloud.sh b/Tests/scripts/uninstall_packs_and_reset_bucket_cloud.sh index 4152721a228..57bcba4d1d3 100644 --- a/Tests/scripts/uninstall_packs_and_reset_bucket_cloud.sh +++ b/Tests/scripts/uninstall_packs_and_reset_bucket_cloud.sh @@ -7,12 +7,12 @@ CLOUD_SERVERS_PATH=$(cat $CLOUD_SERVERS_FILE) echo ${CLOUD_API_KEYS} > "cloud_api_keys.json" if [[ -z ${CLOUD_CHOSEN_MACHINE_ID} ]]; then - echo "CLOUD_CHOSEN_MACHINE_ID is not defiened, exiting..." + echo "CLOUD_CHOSEN_MACHINE_ID is not defined, exiting..." else gcloud auth activate-service-account --key-file="$GCS_MARKET_KEY" > auth.out 2>&1 echo "Copying prod bucket to $CLOUD_CHOSEN_MACHINE_ID bucket." gsutil -m cp -r "gs://$GCS_SOURCE_BUCKET/content" "gs://$GCS_MACHINES_BUCKET/$CLOUD_CHOSEN_MACHINE_ID/" > "$ARTIFACTS_FOLDER/Copy_prod_bucket_to_cloud_machine_cleanup.log" 2>&1 echo "sleeping 120 seconds" sleep 120 - python3 ./Tests/Marketplace/search_and_uninstall_pack.py --cloud_machine $CLOUD_CHOSEN_MACHINE_ID --cloud_servers_path $CLOUD_SERVERS_PATH --cloud_servers_api_keys "cloud_api_keys.json" --unremovable_packs $UNREMOVABLE_PACKS --one-by-one + python3 ./Tests/Marketplace/search_and_uninstall_pack.py --cloud_machine $CLOUD_CHOSEN_MACHINE_ID --cloud_servers_path $CLOUD_SERVERS_PATH --cloud_servers_api_keys "cloud_api_keys.json" --unremovable_packs $UNREMOVABLE_PACKS --one-by-one --build-number "$CI_PIPELINE_ID" fi