Jira token auth

.gitignore

@@ -141,3 +141,4 @@ needs.rst
 
 !/presentations/_static/dynamic_background/js/lib
 /needs_save.json
+!/tests/roots/test-sne-import/needs.json

docs/changelog.rst

@@ -5,6 +5,10 @@ Changelog
 -----
 **Released**: under developement
 
+* Improvement: Added pagination and retry attemps for codebeamer API requests
+* Improvement: Added bearer auth
+
+
 1.0.1
 -----
 **Released**: 01.02.2023

docs/conf.py

@@ -15,9 +15,10 @@
 import os
 import sys
 
+from sphinx_needs_enterprise.version import VERSION
+
 # from docutils.parsers.rst import directives
 
-from sphinx_needs_enterprise.version import VERSION
 
 sys.path.insert(0, os.path.abspath("../sphinx_needs_enterprise"))
 
@@ -303,4 +304,4 @@ def setup(app):
     }
 }
 
-linkcheck_workers = 5
+linkcheck_workers = 5

docs/requirements.txt

@@ -1,7 +1,8 @@
 sphinx>=3.4
 sphinxcontrib-plantuml
-sphinx-needs==1.0.1
+sphinx-needs==1.2.2
 sphinx-copybutton
+pygls
 
 # Sphinx-Material Theme + Fix to support Sphinx-Needs
 sphinx-design

docs/services/jira.rst

@@ -52,6 +52,15 @@ notation.
 
 See also :ref:`conf_query` for more details.
 
+.. _jira_enable_bearer_auth:
+
+enable_bearer_auth
+~~~~~~~~~~~~~~~~~~
+
+**Default** False
+
+If specified, enables bearer auth for jira API service. See `Jira API docs https://confluence.atlassian.com/enterprise/using-personal-access-tokens-1026032365.html` for more details.
+
 
 id_prefix
 ~~~~~~~~~

presentations/requirements.txt

@@ -1,4 +1,5 @@
 sphinx
-sphinx-needs==1.0.1
+sphinx-needs==1.2.2 
 sphinxcontrib-plantuml
 sphinx-revealjs
+pygls<=1.0

pyproject.toml

@@ -12,7 +12,7 @@ documentation = "https://useblocks.com/sphinx-needs-enterprise"
 [tool.poetry.dependencies]
 python = ">3.7,<4.0"
 sphinx = ">=4.2"
-sphinx-needs = ">=1.0.1"
+sphinx-needs = ">=1.2.2"
 licensing = ">=0.31"
 requests = ">=2"
 jinja2 = ">=2"
@@ -50,7 +50,7 @@ line-length = 120
 profile = "black"
 
 [tool.poetry.extras]
-docs = ["sphinx>=4.0"]
+docs = ["sphinx"]
 
 [build-system]
 requires = ["poetry>=0.12"]

sphinx_needs_enterprise/extensions/extension.py

@@ -14,6 +14,15 @@
 from sphinx_needs_enterprise.util import dict_get, jinja_parse
 
 
+class BearerAuth(requests.auth.AuthBase):
+    def __init__(self, token):
+        self.token = token
+
+    def __call__(self, r):
+        r.headers["authorization"] = "Bearer " + self.token
+        return r
+
+
 class ServiceExtension(BaseService):
     def __init__(
         self,
@@ -31,6 +40,7 @@ def __init__(
         mappings_replaces=None,
         extra_data=None,
         ssl_location=None,
+        bearer_auth=None,
         **kwargs,
     ):
 
@@ -55,6 +65,7 @@ def __init__(
         self.extra_data = extra_data or config.get("extra_data", {})
 
         self.ssl_location = ssl_location or config.get("ssl_cert_abspath", "")
+        self.bearer_auth = ssl_location or config.get("enable_bearer_auth", "")
 
         self.license_key = None
         self.product_id = None
@@ -108,7 +119,16 @@ def _prepare_request(self, options):
         url = options.get("url", self.url)
         url = url + self.url_postfix
 
-        auth = (options.get("user", self.user), options.get("password", self.password))
+        bearer_auth = options.get("enable_bearer_auth", self.bearer_auth)
+
+        if bearer_auth:
+            print("using bearer auth")
+
+            auth = BearerAuth(options.get("password", self.password))
+
+        else:
+
+            auth = (options.get("user", self.user), options.get("password", self.password))
         query = options.get("query", self.query)
         query = query + self.query_postfix
 
@@ -145,7 +165,30 @@ def _send_request(self, request, cert_abspath=""):
             result = requests.request(**request)
 
         if result.status_code >= 300:
-            raise CommunicationException(f"Problems accessing {result.url}.\nReason: {result.text}")
+            from time import sleep
+
+            delay = 15
+            retry_limit = 3
+            retries = 0
+
+            print(f"HTTP status code {result.status_code}")
+            print(f"retrying {retry_limit} times with {delay} s delay")
+
+            while result.status_code != 200 and retries < retry_limit:
+                retries += 1
+                print(f"retry #{retries} in {delay} seconds")
+                sleep(delay)
+                print(f"HTTP status code {result.status_code}")
+
+                if cert_abspath:
+                    result = requests.request(**request, verify=cert_abspath)
+
+                else:
+                    result = requests.request(**request)
+
+            if result.status_code >= 300:
+
+                raise CommunicationException(f"Problems accessing {result.url}.\nReason: {result.text}")
 
         return result
 
@@ -215,8 +258,30 @@ def _extract_data(self, data, options):
                 if name == "id":
                     need_values[name] = prefix + need_values.get(name, "")
 
+            if "title" not in need_values.keys():
+                need_values["title"] = "no title provided"
+
             finale_data = {"content": content}
             finale_data.update(need_values)
 
             need_data.append(finale_data)
         return need_data
+
+    def replace_content(self, options, data):
+        replace_dict = options.get("replace_content", self.config["replace_content"])
+        options["replace_content"] = replace_dict
+
+        if not replace_dict:
+            return data
+
+        # TODO invert logic, first parse datum in data, then replace if present, saves compute time
+
+        for field, replacements in replace_dict.items(): # entries in service_config["replace_content"], low amount
+            for string, replacement in replacements.items(): # string to replace
+                for datum in data: # entries
+
+                    if field in datum.keys():
+
+                        datum[field] = datum[field].replace(string, replacement)
+
+        return data

sphinx_needs_enterprise/scripts/cli.py

@@ -7,6 +7,7 @@
 import click
 import elasticsearch
 import jinja2
+import requests
 from tqdm import tqdm
 
 # API has changed with Sphinx-Needs version 1.0.1
@@ -18,6 +19,15 @@
 from sphinx_needs_enterprise.scripts.loader import service_loader
 
 
+class BearerAuth(requests.auth.AuthBase):
+    def __init__(self, token):
+        self.token = token
+
+    def __call__(self, r):
+        r.headers["authorization"] = "Bearer " + self.token
+        return r
+
+
 @click.group()
 def cli():
     pass
@@ -55,6 +65,9 @@ def import_cmd(service, conf, outdir, query, old_needfile, version, wipe):
     click.echo("Done")
     click.echo(f"Retrieved {len(data)} elements")
 
+    if "replace_content" in service_obj.config:
+        data = service_obj.replace_content(options, data)
+
     # Storing data
     os.makedirs(outdir, exist_ok=True)
     needlist = NeedsList(sphinx_config, outdir, ".")
@@ -95,7 +108,6 @@ def import_cmd(service, conf, outdir, query, old_needfile, version, wipe):
                 f"Warning: new created needs from imported data do not have such needs options: \
                 {not_included_options}"
             )
-
     for datum in data:
         needlist.add_need(version, datum)
 

sphinx_needs_enterprise/services/codebeamer.py

@@ -74,29 +74,118 @@ def request(self, options=None):
 
         params = self._prepare_request(options)
 
+        delay = 1
+        current_page = 1
+        retry_limit = 3
+
+        # query 250 objects at once
         request_params = {
             "method": "GET",
             "url": params["url"],
             "auth": params["auth"],
             "params": {
                 "queryString": params["query"],
+                "pageSize": 250,
+                "page": current_page,
                 "descriptionFormat": "HTML",
                 "descFormat": "HTML",
             },
         }
-        answer = self._send_request(request_params, params["cert_abspath"])
-        data = answer.json()["items"]
+        result = self._send_request(request_params, params["cert_abspath"])
+
+        response_json = result.json()
+
+        combined_objects = response_json["items"]
+        print(len(combined_objects))
+
+        # if first request fails, try RETRY_LIMIT times again
+        retries = 0
+        if result.status_code != 200:
+            while retries < retry_limit:
+                print("retrying connection")
+                result = self._send_request(request_params, params["cert_abspath"])
+                retries += 1
+                time.sleep(3)
+
+        # check return code
+
+        if result.status_code == 200:
+
+            print("start pagination")
+
+            total = response_json["total"]
+            page_size = response_json["pageSize"]
+
+            # there are more items than shown, request more pages
+            if total > page_size:
+
+                # minimum amount of pages needed for example if pageSize = 100 and 1000 objects
+                min_pages = total // page_size
+
+                # if page_size multiple of amount of objects, no additional query needed
+                if total % page_size == 0:
+
+                    total_page_count = min_pages
+
+                else:
+                    # one more pagination request needed to get remainder of data
+                    total_page_count = min_pages + 1
+
+                # request pages 2 - last page
+                for i in range(total_page_count):
+
+                    time.sleep(delay)
+
+                    current_page += 1
+                    request_params["params"]["page"] = current_page
+
+                    print(f"querying page {current_page}")
+
+                    result = self._send_request(request_params, params["cert_abspath"])
+
+                    status = result.status_code
+
+                    retries = 0
+
+                    if status != 200:
+
+                        while retries < retry_limit:
+                            print("retrying connection")
+                            result = self._send_request(request_params, params["cert_abspath"])
+                            retries += 1
+                            time.sleep(3)
+
+                    response_json = result.json()
+
+                    [combined_objects.append(item) for item in response_json["items"]]
+
+                    print(len(combined_objects))
+
+                    retries += 1
+
+        # print("starting wiki2html requests")
+        # print(len(combined_objects))
+
+        data = combined_objects
+
         for datum in data:
-            delay = cb_request_delay_ms / 1000
-            if delay:
-                time.sleep(delay)
 
             # Be sure "description" is set and valid
             if "description" not in datum or datum["description"] is None:
                 datum["description"] = ""
             elif datum["descriptionFormat"] == "Wiki" and wiki2html == "True":
                 # Transform the Codebeamer wiki syntax to HTML.
                 # Must be done by an API request for each item.
+
+                print("format is WIKI")
+                print("pre translation")
+                print("--------------------")
+                print(datum["description"])
+
+                delay = cb_request_delay_ms / 1000
+                if delay:
+                    time.sleep(delay)
+
                 url = options.get("url", self.url)
                 wiki2html_id = options.get("wiki2html_id", 2)
 
@@ -114,9 +203,15 @@ def request(self, options=None):
 
                 wiki2html_answer = self._send_request(wiki2html_params, params["cert_abspath"])
                 datum["description"] = wiki2html_answer.text
+
+                print("--------------------")
+                print("post translation")
+                print(datum["description"])
 
         need_data = self._extract_data(data, options)
 
+        print(len(need_data))
+
         return need_data
 
     def debug(self, options):