[clients] - Refactor classes not to use pydantic (#3985)

* [clients] - Check health on both xsiam/xsoar-saas

* fix client tests

* changelog

* validate health of xsoar6 too

* align env vars + raise unhealthy exceptions

* update changelog

* fix existing tests

* some fixes

* pydantic to normal objects

* config

* add _str__ to xsoar-clients

* implement is_server_type

* refactor get_client_from_server_type

* update get_from_server_type

* ServerAbout class

* fix most of tests

* if no auth_id - do not validate

* add tests

* remove build num property

* refactor get_server_type

* logs improvments

* logs

* test_get_xsoar_on_prem_client_from_server_type_with_auth_id test

* update __str__ and fix some bugs

* warning if /about is not retrievable

* changelog

* fix tests

.changelog/3985.yml

@@ -0,0 +1,4 @@
+changes:
+- description: Added support to check the health status when initialing a client for xsoar-on-prem/xsoar-saas by default, will throw exception if server is not healthy.
+  type: feature
+pr_number: 3985

demisto_sdk/commands/common/clients/__init__.py

@@ -2,16 +2,21 @@
 from functools import lru_cache
 from typing import Optional
 
-import demisto_client
-import pydantic
+from urllib3.exceptions import MaxRetryError
 
 from demisto_sdk.commands.common.clients.configs import (
     XsiamClientConfig,
     XsoarClientConfig,
     XsoarSaasClientConfig,
 )
+from demisto_sdk.commands.common.clients.errors import (
+    InvalidServerType,
+)
 from demisto_sdk.commands.common.clients.xsiam.xsiam_api_client import XsiamClient
-from demisto_sdk.commands.common.clients.xsoar.xsoar_api_client import XsoarClient
+from demisto_sdk.commands.common.clients.xsoar.xsoar_api_client import (
+    ServerType,
+    XsoarClient,
+)
 from demisto_sdk.commands.common.clients.xsoar_saas.xsoar_saas_api_client import (
     XsoarSaasClient,
 )
@@ -24,6 +29,7 @@
     DEMISTO_VERIFY_SSL,
     MarketplaceVersions,
 )
+from demisto_sdk.commands.common.logger import logger
 from demisto_sdk.commands.common.tools import string_to_bool
 
 
@@ -48,10 +54,12 @@ def get_client_from_config(client_config: XsoarClientConfig) -> XsoarClient:
 
 def get_client_from_marketplace(
     marketplace: MarketplaceVersions,
-    base_url: Optional[str] = os.getenv(DEMISTO_BASE_URL, ""),
-    api_key: Optional[str] = os.getenv(DEMISTO_KEY, ""),
-    auth_id: Optional[str] = os.getenv(AUTH_ID, ""),
-    verify_ssl: bool = string_to_bool(os.getenv(DEMISTO_VERIFY_SSL, False)),
+    base_url: Optional[str] = None,
+    api_key: Optional[str] = None,
+    auth_id: Optional[str] = None,
+    username: Optional[str] = None,
+    password: Optional[str] = None,
+    verify_ssl: Optional[bool] = None,
 ) -> XsoarClient:
     """
     Returns the client based on the marketplace.
@@ -61,41 +69,54 @@ def get_client_from_marketplace(
         base_url: the base URL, if not provided will take from DEMISTO_BASE_URL env var
         api_key: the api key, if not provided will take from DEMISTO_API_KEY env var
         auth_id: the auth ID, if not provided will take from XSIAM_AUTH_ID env var
+        username: the username to authenticate, relevant only for xsoar on prem
+        password: the password to authenticate, relevant only for xsoar on prem
         verify_ssl: whether in each request SSL should be verified, True if yes, False if not
                     if verify_ssl = None, will take the SSL verification from DEMISTO_VERIFY_SSL env var
 
     Returns:
         the correct client according to the marketplace provided
     """
+    _base_url = base_url or os.getenv(DEMISTO_BASE_URL, "")
+    _api_key = api_key or os.getenv(DEMISTO_KEY, "")
+    _auth_id = auth_id or os.getenv(AUTH_ID)
+    _username = username or os.getenv(DEMISTO_USERNAME, "")
+    _password = password or os.getenv(DEMISTO_PASSWORD, "")
+    _verify_ssl = (
+        verify_ssl
+        if verify_ssl is not None
+        else string_to_bool(os.getenv(DEMISTO_VERIFY_SSL, False))
+    )
+
     if marketplace in (MarketplaceVersions.XSOAR_ON_PREM, MarketplaceVersions.XSOAR):
         config = XsoarClientConfig(
-            base_api_url=base_url, api_key=api_key, verify_ssl=verify_ssl
+            base_api_url=_base_url, api_key=_api_key, verify_ssl=_verify_ssl
         )
     elif marketplace == MarketplaceVersions.XSOAR_SAAS:
         config = XsoarSaasClientConfig(
-            base_api_url=base_url,
-            api_key=api_key,
-            auth_id=auth_id,
-            verify_ssl=verify_ssl,
+            base_api_url=_base_url,
+            api_key=_api_key,
+            auth_id=_auth_id,
+            verify_ssl=_verify_ssl,
         )
     else:
         config = XsiamClientConfig(
-            base_api_url=base_url,
-            api_key=api_key,
-            auth_id=auth_id,
-            verify_ssl=verify_ssl,
+            base_api_url=_base_url,
+            api_key=_api_key,
+            auth_id=_auth_id,
+            verify_ssl=_verify_ssl,
         )
     return get_client_from_config(config)
 
 
 @lru_cache
 def get_client_from_server_type(
-    base_url: Optional[str] = os.getenv(DEMISTO_BASE_URL, ""),
-    api_key: Optional[str] = os.getenv(DEMISTO_KEY, ""),
-    auth_id: Optional[str] = os.getenv(AUTH_ID, ""),
-    username: Optional[str] = os.getenv(DEMISTO_USERNAME, ""),
-    password: Optional[str] = os.getenv(DEMISTO_PASSWORD, ""),
-    verify_ssl: bool = string_to_bool(os.getenv(DEMISTO_VERIFY_SSL, False)),
+    base_url: Optional[str] = None,
+    api_key: Optional[str] = None,
+    auth_id: Optional[str] = None,
+    username: Optional[str] = None,
+    password: Optional[str] = None,
+    verify_ssl: Optional[bool] = None,
 ) -> XsoarClient:
     """
     Returns the client based on the server type by doing api requests to determine which server it is
@@ -112,75 +133,80 @@ def get_client_from_server_type(
     Returns:
         the correct client based on querying the type of the server
     """
-    _client = demisto_client.configure(
-        base_url=base_url,
-        api_key=api_key,
-        verify_ssl=verify_ssl,
-        username=username,
-        password=password,
+    _base_url = base_url or os.getenv(DEMISTO_BASE_URL, "")
+    _api_key = api_key or os.getenv(DEMISTO_KEY, "")
+    _auth_id = auth_id or os.getenv(AUTH_ID)
+    _username = username or os.getenv(DEMISTO_USERNAME, "")
+    _password = password or os.getenv(DEMISTO_PASSWORD, "")
+    _verify_ssl = (
+        verify_ssl
+        if verify_ssl is not None
+        else string_to_bool(os.getenv(DEMISTO_VERIFY_SSL, False))
     )
 
-    about = XsoarClient.get_xsoar_about(_client)
-    product_mode = about.get("productMode")
-    deployment_mode = about.get("deploymentMode")
-    server_version = about.get("demistoVersion")
+    if not _auth_id and (_api_key or (_username and _password)):
+        # if no auth-id is provided, it must be xsoar-on-prem
+        logger.debug(
+            f"Assuming {_base_url} is {ServerType.XSOAR} server as {AUTH_ID} is not defined"
+        )
+        return XsoarClient(
+            config=XsoarClientConfig(
+                base_api_url=_base_url,
+                api_key=_api_key,
+                user=_username,
+                password=_password,
+                verify_ssl=_verify_ssl,
+            ),
+        )
 
-    if XsiamClient.is_xsiam(_client, product_mode=product_mode):
+    should_validate_server_type = True
+    logger.debug(f"Checking if {_base_url} is {ServerType.XSIAM}")
+
+    try:
         return XsiamClient(
             config=XsiamClientConfig(
-                base_api_url=base_url,
-                api_key=api_key,
-                auth_id=auth_id,
-                verify_ssl=verify_ssl,
-                about=about,
+                base_api_url=_base_url,
+                api_key=_api_key,
+                auth_id=_auth_id,
+                verify_ssl=_verify_ssl,
+            ),
+            should_validate_server_type=should_validate_server_type,
+        )
+    except (InvalidServerType, MaxRetryError):
+        logger.debug(f"Checking if {_base_url} is {ServerType.XSOAR_SAAS}")
+
+    try:
+        return XsoarSaasClient(
+            config=XsoarSaasClientConfig(
+                base_api_url=_base_url,
+                api_key=_api_key,
+                auth_id=_auth_id,
+                verify_ssl=_verify_ssl,
             ),
-            client=_client,
-            about_xsoar=about,
+            should_validate_server_type=should_validate_server_type,
         )
+    except (InvalidServerType, MaxRetryError):
+        logger.debug(f"Checking if {_base_url} is {ServerType.XSOAR}")
 
-    if XsoarSaasClient.is_xsoar_saas(
-        server_version, product_mode=product_mode, deployment_mode=deployment_mode
-    ):
-        try:
-            return XsoarSaasClient(
-                config=XsoarSaasClientConfig(
-                    base_api_url=base_url,
-                    api_key=api_key,
-                    auth_id=auth_id,
-                    verify_ssl=verify_ssl,
-                ),
-                client=_client,
-                about=about,
-            )
-        except pydantic.ValidationError:
-            # xsoar-on-prem that can have server version > 8.0.0
-            return XsoarClient(
-                config=XsoarClientConfig(
-                    base_api_url=base_url,
-                    api_key=api_key,
-                    auth_id=auth_id,
-                    user=username,
-                    password=password,
-                    verify_ssl=verify_ssl,
-                ),
-                client=_client,
-                about=about,
-            )
-
-    if XsoarClient.is_xsoar_on_prem(
-        server_version, product_mode=product_mode, deployment_mode=deployment_mode
-    ):
+    try:
+        # if xsiam-auth-id is defined by mistake
+        os.environ.pop(AUTH_ID, None)
         return XsoarClient(
             config=XsoarClientConfig(
-                base_api_url=base_url,
-                api_key=api_key,
-                auth_id=auth_id,
-                user=username,
-                password=password,
-                verify_ssl=verify_ssl,
+                base_api_url=_base_url,
+                api_key=_api_key,
+                user=_username,
+                password=_password,
+                verify_ssl=_verify_ssl,
             ),
-            client=_client,
-            about=about,
+            should_validate_server_type=should_validate_server_type,
         )
-
-    raise RuntimeError(f"Could not determine the correct api-client for {base_url}")
+    except Exception as error:
+        logger.debug(
+            f"The {_base_url} is not {ServerType.XSOAR} server, error: {error}"
+        )
+        logger.error(
+            f"Could not determine the correct api-client for {_base_url}, "
+            f"make sure the {DEMISTO_BASE_URL}, {DEMISTO_KEY}, {AUTH_ID} are defined properly"
+        )
+        raise

demisto_sdk/commands/common/clients/errors.py

@@ -4,3 +4,13 @@
 class UnAuthorized(RuntimeError):
     def __init__(self, message: str, status_code: int = requests.codes.unauthorized):
         super().__init__(f"Unauthorized: {message} - code {status_code}")
+
+
+class UnHealthyServer(RuntimeError):
+    def __init__(self, server: str):
+        super().__init__(f"The {server} is unhealthy")
+
+
+class InvalidServerType(ValueError):
+    def __init__(self, server: str, server_type: str):
+        super().__init__(f"The server {server} is not {server_type} server")