Uninstall one-by-one with waiting for the status from the server not …

…to be in "create / update / delete operation is already in progress (10102)" (demisto#28586)

.gitignore

@@ -93,6 +93,7 @@ test_runner.sh
 # log files
 demisto_sdk_debug.log
 demisto_sdk_debug.log.*
+*.log
 
 # Ignore Modeling Rules test conf
 Packs/**/ModelingRules/**/**/*_testdata.json

.gitlab/ci/.gitlab-ci.bucket-upload.yml

@@ -221,18 +221,18 @@ install-packs-in-server-master:
     - CLOUD_SERVERS_PATH=$(cat $CLOUD_SERVERS_FILE)
     - echo ${CLOUD_API_KEYS} > "cloud_api_keys.json"
 
+    - section_start "Clean Machine"
+    - ./Tests/scripts/uninstall_packs_and_reset_bucket_cloud.sh || EXIT_CODE=$?
+    - section_end "Clean Machine"
+
     - |
       if [ "$INSTANCE_ROLE" == XSIAM ]; then
         section_start "Run end to end sanity tests"
-        
+      
         python3 -m pytest ./Tests/tests_end_to_end_xsiam -v --cloud_machine "$CLOUD_CHOSEN_MACHINE_ID" --cloud_servers_path "$CLOUD_SERVERS_PATH" --cloud_servers_api_keys "cloud_api_keys.json" --disable-warnings
         section_end "Run end to end sanity tests"
       fi
 
-    - section_start "Clean Machine"
-    - ./Tests/scripts/uninstall_packs_and_reset_bucket_cloud.sh
-    - section_end "Clean Machine"
-
     - section_start "Get Instance Variables"
     - echo INSTANCE_ROLE="$INSTANCE_ROLE"
     - echo INSTANCE_CREATED="$INSTANCE_CREATED"

Tests/Marketplace/search_and_uninstall_pack.py

@@ -1,15 +1,21 @@
-import ast
 import argparse
+import ast
 import math
 import os
 import sys
+from datetime import datetime, timedelta
+from time import sleep
 
 import demisto_client
-from Tests.configure_and_test_integration_instances import CloudBuild
+from demisto_client.demisto_api.rest import ApiException
+from urllib3.exceptions import HTTPWarning, HTTPError
+
+from Tests.Marketplace.configure_and_install_packs import search_and_install_packs_and_their_dependencies
+from Tests.configure_and_test_integration_instances import CloudBuild, get_custom_user_agent
 from Tests.scripts.utils import logging_wrapper as logging
 from Tests.scripts.utils.log_util import install_logging
-from Tests.Marketplace.configure_and_install_packs import search_and_install_packs_and_their_dependencies
-from time import sleep
+
+ALREADY_IN_PROGRESS = "create / update / delete operation is already in progress (10102)"
 
 
 def get_all_installed_packs(client: demisto_client, unremovable_packs: list):
@@ -77,23 +83,140 @@ def uninstall_all_packs_one_by_one(client: demisto_client, hostname, unremovable
     return uninstalled_count == len(packs_to_uninstall)
 
 
-def uninstall_pack(client: demisto_client, pack_id: str):
+def get_updating_status(client: demisto_client,
+                        attempts_count: int = 5,
+                        sleep_interval: int = 60,
+                        ) -> tuple[bool, bool | None]:
+    try:
+        for attempt in range(attempts_count - 1, -1, -1):
+            try:
+                logging.info(f"Getting installation/update status, Attempt: {attempts_count - attempt}/{attempts_count}")
+                response, status_code, headers = demisto_client.generic_request_func(client,
+                                                                                     path='/content/updating',
+                                                                                     method='GET',
+                                                                                     accept='application/json',
+                                                                                     _request_timeout=None)
+
+                if 200 <= status_code < 300 and status_code != 204:
+                    # the endpoint simply returns a string (rather than a json object.)
+                    updating_status = 'true' in str(response).lower()
+                    logging.info(f"Got updating status: {updating_status}")
+                    return True, updating_status
+                else:
+                    logging.info(f"Got bad response for updating status: {response}")
+
+                if not attempt:
+                    raise Exception(f"Got bad status code: {status_code}, headers: {headers}")
+
+                logging.warning(f"Got bad status code: {status_code} from the server, headers: {headers}")
+
+            except ApiException as ex:
+                if not attempt:  # exhausted all attempts, understand what happened and exit.
+                    # Unknown exception reason, re-raise.
+                    raise Exception(f"Got status {ex.status} from server, message: {ex.body}, headers: {ex.headers}") from ex
+            except (HTTPError, HTTPWarning) as http_ex:
+                if not attempt:
+                    raise Exception("Failed to perform http request to the server") from http_ex
+
+            # There are more attempts available, sleep and retry.
+            logging.debug(f"Failed to get installation/update status, sleeping for {sleep_interval} seconds.")
+            sleep(sleep_interval)
+
+    except Exception as e:
+        logging.exception(f'The request to get update status has failed. Additional info: {str(e)}')
+    return False, None
+
+
+def wait_until_not_updating(client: demisto_client,
+                            attempts_count: int = 2,
+                            sleep_interval: int = 30,
+                            maximum_time_to_wait: int = 600,
+                            ) -> bool:
     """
 
     Args:
         client (demisto_client): The client to connect to.
-        pack_id: packs id to uninstall
+        attempts_count (int): The number of attempts to install the packs.
+        sleep_interval (int): The sleep interval, in seconds, between install attempts.
+        maximum_time_to_wait (int): The maximum time to wait for the server to exit the updating mode, in seconds.
+    Returns:
+        Boolean - If the operation succeeded.
+
+    """
+    end_time = datetime.utcnow() + timedelta(seconds=maximum_time_to_wait)
+    while datetime.utcnow() <= end_time:
+        success, updating_status = get_updating_status(client)
+        if success:
+            if not updating_status:
+                return True
+            logging.debug(f"Server is still installation/updating status, sleeping for {sleep_interval} seconds.")
+            sleep(sleep_interval)
+        else:
+            if attempts_count := attempts_count - 1:
+                logging.debug(f"failed to get installation/updating status, sleeping for {sleep_interval} seconds.")
+                sleep(sleep_interval)
+            else:
+                logging.info("Exiting after exhausting all attempts")
+                return False
+    logging.info(f"Exiting after exhausting the allowed time:{maximum_time_to_wait} seconds")
+    return False
+
 
+def uninstall_pack(client: demisto_client,
+                   pack_id: str,
+                   attempts_count: int = 5,
+                   sleep_interval: int = 60,
+                   ):
+    """
+
+    Args:
+        client (demisto_client): The client to connect to.
+        pack_id: packs id to uninstall
+        attempts_count (int): The number of attempts to install the packs.
+        sleep_interval (int): The sleep interval, in seconds, between install attempts.
     Returns:
         Boolean - If the operation succeeded.
 
     """
     try:
-        demisto_client.generic_request_func(client,
-                                            path=f'/contentpacks/installed/{pack_id}',
-                                            method='DELETE',
-                                            accept='application/json',
-                                            _request_timeout=None)
+        for attempt in range(attempts_count - 1, -1, -1):
+            try:
+                logging.info(f"Uninstalling packs {pack_id}, Attempt: {attempts_count - attempt}/{attempts_count}")
+                response, status_code, headers = demisto_client.generic_request_func(client,
+                                                                                     path=f'/contentpacks/installed/{pack_id}',
+                                                                                     method='DELETE',
+                                                                                     accept='application/json',
+                                                                                     _request_timeout=None)
+
+                if 200 <= status_code < 300 and status_code != 204:
+                    logging.success(f'Pack: {pack_id} was successfully uninstalled from the server')
+                    break
+
+                if not attempt:
+                    raise Exception(f"Got bad status code: {status_code}, headers: {headers}")
+
+                logging.warning(f"Got bad status code: {status_code} from the server, headers: {headers}")
+
+            except ApiException as ex:
+
+                if ALREADY_IN_PROGRESS in ex.body:
+                    wait_succeeded = wait_until_not_updating(client)
+                    if not wait_succeeded:
+                        raise Exception(
+                            "Failed to wait for the server to exit installation/updating status"
+                        ) from ex
+
+                if not attempt:  # exhausted all attempts, understand what happened and exit.
+                    # Unknown exception reason, re-raise.
+                    raise Exception(f"Got {ex.status} from server, message: {ex.body}, headers: {ex.headers}") from ex
+            except (HTTPError, HTTPWarning) as http_ex:
+                if not attempt:
+                    raise Exception("Failed to perform http request to the server") from http_ex
+
+            # There are more attempts available, sleep and retry.
+            logging.debug(f"failed to uninstall pack: {pack_id}, sleeping for {sleep_interval} seconds.")
+            sleep(sleep_interval)
+
         return True
     except Exception as e:
         logging.exception(f'The request to uninstall packs has failed. Additional info: {str(e)}')
@@ -235,6 +358,7 @@ def options_handler():
     parser.add_argument('--cloud_servers_api_keys', help='Path to the file with cloud Servers api keys.')
     parser.add_argument('--unremovable_packs', help='List of packs that cant be removed.')
     parser.add_argument('--one-by-one', help='Uninstall pack one pack at a time.', action='store_true')
+    parser.add_argument('--build-number', help='CI job number where the instances were created', required=True)
 
     options = parser.parse_args()
 
@@ -244,7 +368,7 @@ def options_handler():
 def main():
     install_logging('cleanup_cloud_instance.log', logger=logging)
 
-    # in cloud we dont use demisto username
+    # In Cloud, We don't use demisto username
     os.environ.pop('DEMISTO_USERNAME', None)
 
     options = options_handler()
@@ -259,6 +383,9 @@ def main():
                                       verify_ssl=False,
                                       api_key=api_key,
                                       auth_id=xdr_auth_id)
+    client.api_client.user_agent = get_custom_user_agent(options.build_number)
+    logging.debug(f'Setting user agent on client to: {client.api_client.user_agent}')
+
     # We are syncing marketplace since we are copying production bucket to build bucket and if packs were configured
     # in earlier builds they will appear in the bucket as it is cached.
     sync_marketplace(client=client)