feat: Migrate Httplib2 to requests library

BREAKING CHANGE: The 'build_http_connection' function is removed. User can directly use the 'requests.request' function

pyproject.toml

@@ -25,8 +25,8 @@ repository = "https://github.com/splunk/addonfactory-ta-library-python"
 [tool.poetry.dependencies]
 python = "^3.7"
 sortedcontainers = "^2"
-httplib2 = "^0"
 defusedxml = "^0"
+requests = "^2.26.0"
 
 [tool.poetry.dev-dependencies]
 pytest = "^6"

splunktalib/common/xml_dom_parser.py

@@ -23,7 +23,6 @@ def parse_conf_xml_dom(xml_content):
     """
     @xml_content: XML DOM from splunkd
     """
-    xml_content = xml_content.decode("utf-8")
     m = re.search(r'xmlns="([^"]+)"', xml_content)
     ns = m.group(1)
     m = re.search(r'xmlns:s="([^"]+)"', xml_content)

splunktalib/conf_manager/request.py

@@ -36,30 +36,28 @@ def content_request(uri, session_key, method, payload, err_msg):
     ConfRequestException
     """
 
-    resp, content = rest.splunkd_request(
-        uri, session_key, method, data=payload, retry=3
-    )
-    if resp is None and content is None:
+    resp = rest.splunkd_request(uri, session_key, method, data=payload, retry=3)
+    if resp is None:
         return None
 
-    if resp.status >= 200 and resp.status <= 204:
-        return content
+    if resp.status_code >= 200 and resp.status_code <= 204:
+        return resp.text
     else:
         msg = "{}, status={}, reason={}, detail={}".format(
             err_msg,
-            resp.status,
+            resp.status_code,
             resp.reason,
-            content.decode("utf-8"),
+            resp.text,
         )
 
-        if not (method == "GET" and resp.status == 404):
+        if not (method == "GET" and resp.status_code == 404):
             log.logger.error(msg)
 
-        if resp.status == 404:
+        if resp.status_code == 404:
             raise ConfNotExistsException(msg)
-        if resp.status == 409:
+        if resp.status_code == 409:
             raise ConfExistsException(msg)
         else:
-            if content and "already exists" in content:
+            if resp.text and "already exists" in resp.text:
                 raise ConfExistsException(msg)
             raise ConfRequestException(msg)

splunktalib/credentials.py

@@ -104,14 +104,12 @@ def get_session_key(username, password, splunkd_uri="https://localhost:8089"):
             "password": password,
         }
 
-        response, content = rest.splunkd_request(
-            eid, None, method="POST", data=postargs
-        )
+        response = rest.splunkd_request(eid, None, method="POST", data=postargs)
 
-        if response is None and content is None:
+        if response is None:
             raise CredException("Get session key failed.")
 
-        xml_obj = xdm.parseString(content)
+        xml_obj = xdm.parseString(response.text)
         session_nodes = xml_obj.getElementsByTagName("sessionKey")
         if not session_nodes:
             raise CredException("Invalid username or password.")
@@ -165,13 +163,13 @@ def _do_update(self, name, password):
         except CredException:
             payload = {"password": password}
             endpoint = self._get_endpoint(name)
-            response, _ = rest.splunkd_request(
+            response = rest.splunkd_request(
                 endpoint, self._session_key, method="POST", data=payload
             )
-            if not response or response.status not in (200, 201):
+            if not response or response.status_code not in (200, 201):
                 raise CredException(
                     "Unable to update password for username={}, status={}".format(
-                        name, response.status
+                        name, response.status_code
                     )
                 )
 
@@ -188,10 +186,10 @@ def _create(self, name, str_to_encrypt):
         }
 
         endpoint = self._get_endpoint(name)
-        resp, content = rest.splunkd_request(
+        resp = rest.splunkd_request(
             endpoint, self._session_key, method="POST", data=payload
         )
-        if not resp or resp.status not in (200, 201, "200", "201"):
+        if not resp or resp.status_code not in (200, 201, "200", "201"):
             raise CredException("Failed to encrypt username {}".format(name))
 
     def delete(self, name, throw=False):
@@ -236,14 +234,12 @@ def _delete(self, name, throw=False):
         """
 
         endpoint = self._get_endpoint(name)
-        response, content = rest.splunkd_request(
-            endpoint, self._session_key, method="DELETE"
-        )
+        response = rest.splunkd_request(endpoint, self._session_key, method="DELETE")
 
-        if response is not None and response.status in (404, "404"):
+        if response is not None and response.status_code in (404, "404"):
             if throw:
                 raise CredNotFound("Credential stanza not exits - {}".format(name))
-        elif not response or response.status not in (200, 201, "200", "201"):
+        elif not response or response.status_code not in (200, 201, "200", "201"):
             if throw:
                 raise CredException(
                     "Failed to delete credential stanza {}".format(name)
@@ -312,11 +308,13 @@ def _get_all_passwords(self):
         """
 
         endpoint = self._get_endpoint()
-        response, content = rest.splunkd_request(
-            endpoint, self._session_key, method="GET"
-        )
-        if response and response.status in (200, 201, "200", "201") and content:
-            return xdp.parse_conf_xml_dom(content)
+        response = rest.splunkd_request(endpoint, self._session_key, method="GET")
+        if (
+            response
+            and response.status_code in (200, 201, "200", "201")
+            and response.text
+        ):
+            return xdp.parse_conf_xml_dom(response.text)
         raise CredException("Failed to get credentials")
 
     def get_clear_password(self, name=None):

splunktalib/kv_client.py

@@ -159,17 +159,15 @@ def _do_request(
     ):
         headers = {"Content-Type": content_type}
 
-        resp, content = rest.splunkd_request(
-            uri, self._session_key, method, headers, data
-        )
-        if resp is None and content is None:
+        resp = rest.splunkd_request(uri, self._session_key, method, headers, data)
+        if resp is None:
             raise KVException("Failed uri={}, data={}".format(uri, data))
 
-        if resp.status in (200, 201):
-            return content
-        elif resp.status == 409:
+        if resp.status_code in (200, 201):
+            return resp.text
+        elif resp.status_code == 409:
             raise KVAlreadyExists("{}-{} already exists".format(uri, data))
-        elif resp.status == 404:
+        elif resp.status_code == 404:
             raise KVNotExists("{}-{} not exists".format(uri, data))
         else:
             raise KVException(

splunktalib/rest.py

@@ -15,23 +15,25 @@
 #
 
 import json
-import urllib.error
 import urllib.parse
-import urllib.request
 from traceback import format_exc
 
-from httplib2 import Http, ProxyInfo, socks
+import requests
 
 import splunktalib.common.log as log
 import splunktalib.common.util as scu
 
 
 def splunkd_request(
-    splunkd_uri, session_key, method="GET", headers=None, data=None, timeout=30, retry=1
-):
-    """
-    :return: httplib2.Response and content
-    """
+    splunkd_uri,
+    session_key,
+    method="GET",
+    headers=None,
+    data=None,
+    timeout=30,
+    retry=1,
+    verify=False,
+) -> requests.Response:
 
     headers = headers if headers is not None else {}
     headers["Authorization"] = "Splunk {}".format(session_key)
@@ -49,104 +51,45 @@ def splunkd_request(
         else:
             data = urllib.parse.urlencode(data)
 
-    http = Http(timeout=timeout, disable_ssl_certificate_validation=True)
     msg_temp = "Failed to send rest request=%s, errcode=%s, reason=%s"
-    resp, content = None, None
+    resp = None
     for _ in range(retry):
         try:
-            resp, content = http.request(
-                splunkd_uri, method=method, headers=headers, body=data
+            resp = requests.request(
+                method=method,
+                url=splunkd_uri,
+                data=data,
+                headers=headers,
+                timeout=timeout,
+                verify=verify,
             )
         except Exception:
             log.logger.error(msg_temp, splunkd_uri, "unknown", format_exc())
         else:
-            if resp.status not in (200, 201):
-                if not (method == "GET" and resp.status == 404):
+            if resp.status_code not in (200, 201):
+                if not (method == "GET" and resp.status_code == 404):
                     log.logger.debug(
-                        msg_temp, splunkd_uri, resp.status, code_to_msg(resp, content)
+                        msg_temp, splunkd_uri, resp.status_code, code_to_msg(resp)
                     )
             else:
-                return resp, content
+                return resp
     else:
-        return resp, content
+        return resp
 
 
-def code_to_msg(resp, content):
+def code_to_msg(response: requests.Response):
     code_msg_tbl = {
-        400: "Request error. reason={}".format(content),
+        400: "Request error. reason={}".format(response.text),
         401: "Authentication failure, invalid access credentials.",
         402: "In-use license disables this feature.",
         403: "Insufficient permission.",
         404: "Requested endpoint does not exist.",
-        409: "Invalid operation for this endpoint. reason={}".format(content),
-        500: "Unspecified internal server error. reason={}".format(content),
+        409: "Invalid operation for this endpoint. reason={}".format(response.text),
+        500: "Unspecified internal server error. reason={}".format(response.text),
         503: (
             "Feature is disabled in the configuration file. "
-            "reason={}".format(content)
+            "reason={}".format(response.text)
         ),
     }
 
-    return code_msg_tbl.get(resp.status, content)
-
-
-def build_http_connection(config, timeout=120, disable_ssl_validation=False):
-    """
-    :config: dict like, proxy and account information are in the following
-             format {
-                 "username": xx,
-                 "password": yy,
-                 "proxy_url": zz,
-                 "proxy_port": aa,
-                 "proxy_username": bb,
-                 "proxy_password": cc,
-                 "proxy_type": http,http_no_tunnel,sock4,sock5,
-                 "proxy_rdns": 0 or 1,
-             }
-    :return: Http2.Http object
-    """
-
-    proxy_type_to_code = {
-        "http": socks.PROXY_TYPE_HTTP,
-        "http_no_tunnel": socks.PROXY_TYPE_HTTP_NO_TUNNEL,
-        "socks4": socks.PROXY_TYPE_SOCKS4,
-        "socks5": socks.PROXY_TYPE_SOCKS5,
-    }
-    if config.get("proxy_type") in proxy_type_to_code:
-        proxy_type = proxy_type_to_code[config["proxy_type"]]
-    else:
-        proxy_type = socks.PROXY_TYPE_HTTP
-
-    rdns = scu.is_true(config.get("proxy_rdns"))
-
-    proxy_info = None
-    if config.get("proxy_url") and config.get("proxy_port"):
-        if config.get("proxy_username") and config.get("proxy_password"):
-            proxy_info = ProxyInfo(
-                proxy_type=proxy_type,
-                proxy_host=config["proxy_url"],
-                proxy_port=int(config["proxy_port"]),
-                proxy_user=config["proxy_username"],
-                proxy_pass=config["proxy_password"],
-                proxy_rdns=rdns,
-            )
-        else:
-            proxy_info = ProxyInfo(
-                proxy_type=proxy_type,
-                proxy_host=config["proxy_url"],
-                proxy_port=int(config["proxy_port"]),
-                proxy_rdns=rdns,
-            )
-    if proxy_info:
-        http = Http(
-            proxy_info=proxy_info,
-            timeout=timeout,
-            disable_ssl_certificate_validation=disable_ssl_validation,
-        )
-    else:
-        http = Http(
-            timeout=timeout, disable_ssl_certificate_validation=disable_ssl_validation
-        )
-
-    if config.get("username") and config.get("password"):
-        http.add_credentials(config["username"], config["password"])
-    return http
+    return code_msg_tbl.get(response.status_code, response.text)

splunktalib/splunk_cluster.py

@@ -20,14 +20,14 @@
 
 
 def _do_rest(uri, session_key):
-    resp, content = rest.splunkd_request(uri, session_key)
+    resp = rest.splunkd_request(uri, session_key)
     if resp is None:
         return None
 
-    if resp.status not in (200, 201):
+    if resp.status_code not in (200, 201):
         return None
 
-    stanza_objs = xdp.parse_conf_xml_dom(content)
+    stanza_objs = xdp.parse_conf_xml_dom(resp.text)
     if not stanza_objs:
         return None