Fix workers failure when cannot report builder log to Pulp

[Korulag]

No description provided.

[github-actions]

Code Coverage Summary

Package	Line Rate
build_node	31%
build_node.builders	24%
build_node.mock	45%
build_node.uploaders	0%
build_node.utils	30%
Summary	31% (1239 / 4053)

View full reports on the Job Summary page

Linter reports

Pylint report

************* Module build_node.build_node_builder
build_node/build_node_builder.py:23:0: E0401: Unable to import 'requests.packages.urllib3.util.retry' (import-error)
build_node/build_node_builder.py:36:0: E0001: Cannot import 'build_node.models' due to 'invalid syntax (build_node.models, line 39)' (syntax-error)
build_node/build_node_builder.py:36:0: E0611: No name 'models' in module 'build_node' (no-name-in-module)
build_node/build_node_builder.py:60:8: R1725: Consider using Python 3 style super() without arguments (super-with-arguments)
build_node/build_node_builder.py:60:52: C0209: Formatting a regular string which could be an f-string (consider-using-f-string)
build_node/build_node_builder.py:64:42: C0209: Formatting a regular string which could be an f-string (consider-using-f-string)
build_node/build_node_builder.py:86:4: R0914: Too many local variables (18/15) (too-many-locals)
build_node/build_node_builder.py:88:32: C0209: Formatting a regular string which could be an f-string (consider-using-f-string)
build_node/build_node_builder.py:135:19: W0718: Catching too general exception Exception (broad-exception-caught)
build_node/build_node_builder.py:155:27: W0718: Catching too general exception Exception (broad-exception-caught)
build_node/build_node_builder.py:175:23: W0718: Catching too general exception Exception (broad-exception-caught)
build_node/build_node_builder.py:184:27: W0718: Catching too general exception Exception (broad-exception-caught)
build_node/build_node_builder.py:209:23: W0718: Catching too general exception Exception (broad-exception-caught)
build_node/build_node_builder.py:86:4: R0912: Too many branches (19/12) (too-many-branches)
build_node/build_node_builder.py:86:4: R0915: Too many statements (86/50) (too-many-statements)
build_node/build_node_builder.py:268:4: R1710: Either all return statements in a function should return an expression, or none of them should. (inconsistent-return-statements)
build_node/build_node_builder.py:361:15: W0718: Catching too general exception Exception (broad-exception-caught)
build_node/build_node_builder.py:355:39: E1101: Instance of 'LookupDict' has no 'conflict' member (no-member)
build_node/build_node_builder.py:334:4: R1710: Either all return statements in a function should return an expression, or none of them should. (inconsistent-return-statements)
build_node/build_node_builder.py:433:35: C0209: Formatting a regular string which could be an f-string (consider-using-f-string)

-----------------------------------
Your code has been rated at 8.22/10

Black report

--- build_node/build_node_builder.py	2024-04-04 14:42:53.128862+00:00
+++ build_node/build_node_builder.py	2024-04-04 14:43:38.324037+00:00
@@ -36,15 +36,13 @@
 from build_node.models import Task
 from build_node.utils.codenotary import notarize_build_artifacts
 
 
 class BuildNodeBuilder(threading.Thread):
-
     """Build thread."""
 
-    def __init__(self, config, thread_num, terminated_event,
-                 graceful_terminated_event):
+    def __init__(self, config, thread_num, terminated_event, graceful_terminated_event):
         """
         Build thread initialization.
 
         Parameters
         ----------
@@ -55,15 +53,13 @@
         terminated_event : threading.Event
             Shows, if process got "kill -15" signal.
         graceful_terminated_event : threading.Event
             Shows, if process got "kill -10" signal.
         """
-        super(BuildNodeBuilder, self).__init__(name='Builder-{0}'.format(
-            thread_num))
+        super(BuildNodeBuilder, self).__init__(name='Builder-{0}'.format(thread_num))
         self.__config = config
-        self.__working_dir = os.path.join(config.working_dir,
-                                          'builder-{0}'.format(thread_num))
+        self.__working_dir = os.path.join(config.working_dir, 'builder-{0}'.format(thread_num))
         self.init_working_dir(self.__working_dir)
         self.__logger = None
         self.__current_task_id = None
         # current task processing start timestamp
         self.__start_ts = None
@@ -72,22 +68,23 @@
         self.__session = None
         self._immudb_wrapper = None
         self._codenotary_enabled = self.__config.codenotary_enabled
         self._build_stats: typing.Optional[typing.Dict[str, typing.Dict[str, str]]] = None
         self._pulp_uploader = PulpRpmUploader(
-            self.__config.pulp_host, self.__config.pulp_user,
-            self.__config.pulp_password, self.__config.pulp_chunk_size,
-            self.__config.pulp_uploader_max_workers
+            self.__config.pulp_host,
+            self.__config.pulp_user,
+            self.__config.pulp_password,
+            self.__config.pulp_chunk_size,
+            self.__config.pulp_uploader_max_workers,
         )
 
         self.__terminated_event = terminated_event
         self.__graceful_terminated_event = graceful_terminated_event
         self.__hostname = platform.node()
 
     def run(self):
-        log_file = os.path.join(self.__working_dir,
-                                'bt-{0}.log'.format(self.name))
+        log_file = os.path.join(self.__working_dir, 'bt-{0}.log'.format(self.name))
         self.__logger = self.init_thread_logger(log_file)
         if self._codenotary_enabled:
             self._immudb_wrapper = ImmudbWrapper(
                 username=self.__config.immudb_username,
                 password=self.__config.immudb_password,
@@ -108,12 +105,11 @@
             self.__current_task_id = task.id
             self.__start_ts = datetime.datetime.utcnow()
             ts = int(self.__start_ts.timestamp())
             task_dir = os.path.join(self.__working_dir, str(task.id))
             artifacts_dir = os.path.join(task_dir, 'artifacts')
-            task_log_file = os.path.join(task_dir,
-                                         f'albs.{task.id}.{ts}.log')
+            task_log_file = os.path.join(task_dir, f'albs.{task.id}.{ts}.log')
             task_log_handler = None
             success = False
             excluded = False
             try:
                 self.__logger.info('processing the task:\n%s', task)
@@ -137,12 +133,11 @@
                     'task %i build failed',
                     task.id,
                 )
                 capture_exception(e)
             finally:
-                only_logs = (not (bool(filter_files(
-                    artifacts_dir, lambda f: f.endswith('.rpm')))))
+                only_logs = not (bool(filter_files(artifacts_dir, lambda f: f.endswith('.rpm'))))
                 notarized_artifacts = {}
                 non_notarized_artifacts = []
                 if self._codenotary_enabled:
                     try:
                         (
@@ -168,25 +163,20 @@
                             'Cannot notarize following artifacts:\n%s',
                             pprint.pformat(non_notarized_artifacts),
                         )
                 build_artifacts = []
                 try:
-                    build_artifacts = self.__upload_artifacts(
-                        artifacts_dir, only_logs=only_logs)
+                    build_artifacts = self.__upload_artifacts(artifacts_dir, only_logs=only_logs)
                 except Exception:
                     self.__logger.exception('Cannot upload task artifacts')
                     build_artifacts = []
                     success = False
                 finally:
                     try:
-                        build_artifacts.append(
-                            self._pulp_uploader.upload_single_file(task_log_file)
-                        )
+                        build_artifacts.append(self._pulp_uploader.upload_single_file(task_log_file))
                     except Exception as e:
-                        self.__logger.exception(
-                            'Cannot upload task log file: %s', str(e)
-                        )
+                        self.__logger.exception('Cannot upload task log file: %s', str(e))
 
                 for artifact in build_artifacts:
                     artifact.cas_hash = notarized_artifacts.get(artifact.path)
 
                 end_ts = datetime.datetime.utcnow()
@@ -199,18 +189,15 @@
                     },
                     **self.__builder.get_build_stats(),
                 })
                 try:
                     if not success and excluded:
-                        self.__report_excluded_task(
-                            task, build_artifacts)
+                        self.__report_excluded_task(task, build_artifacts)
                     else:
-                        self.__report_done_task(
-                            task, success=success, artifacts=build_artifacts)
+                        self.__report_done_task(task, success=success, artifacts=build_artifacts)
                 except Exception:
-                    self.__logger.exception(
-                        'Cannot report task status to the main node')
+                    self.__logger.exception('Cannot report task status to the main node')
                 if task_log_handler:
                     self.__close_task_logger(task_log_handler)
                 self.__current_task_id = None
                 self.__start_ts = None
                 self._build_stats = None
@@ -251,20 +238,18 @@
             Build task working directory path.
         artifacts_dir : str
             Build artifacts storage directory path.
         """
         self.__logger.info('building on the %s node', platform.node())
-        self.__builder = BaseRPMBuilder(self.__config, self.__logger, task,
-                                       task_dir, artifacts_dir,
-                                       self._immudb_wrapper)
+        self.__builder = BaseRPMBuilder(
+            self.__config, self.__logger, task, task_dir, artifacts_dir, self._immudb_wrapper
+        )
         self.__builder.build()
 
     @measure_stage("upload")
-    def __upload_artifacts(self, artifacts_dir,
-                           only_logs: bool = False):
-        artifacts = self._pulp_uploader.upload(
-            artifacts_dir, only_logs=only_logs)
+    def __upload_artifacts(self, artifacts_dir, only_logs: bool = False):
+        artifacts = self._pulp_uploader.upload(artifacts_dir, only_logs=only_logs)
         return artifacts
 
     def __request_task(self):
         supported_arches = [self.__config.base_arch]
         if self.__config.base_arch == 'x86_64':
@@ -320,39 +305,31 @@
             status_forcelist=constants.STATUSES_TO_RETRY,
             method_whitelist=constants.METHODS_TO_RETRY,
             backoff_factor=constants.BACKOFF_FACTOR,
             raise_on_status=True,
         )
-        adapter = requests.adapters.HTTPA

Isort report

--- /build-node/build_node/build_node_builder.py:before	2024-04-04 14:42:53.128862
+++ /build-node/build_node/build_node_builder.py:after	2024-04-04 14:43:38.873099
@@ -10,31 +10,31 @@
 import gzip
 import logging
 import os
-import urllib.parse
 import platform
 import pprint
 import random
 import threading
 import typing
-
-from immudb_wrapper import ImmudbWrapper
+import urllib.parse
+
 import requests
 import requests.adapters
+from immudb_wrapper import ImmudbWrapper
 from requests.packages.urllib3.util.retry import Retry
 from sentry_sdk import capture_exception
 
 from build_node import constants
+from build_node.build_node_errors import BuildError, BuildExcluded
+from build_node.builders.base_builder import measure_stage
 from build_node.builders.base_rpm_builder import BaseRPMBuilder
-from build_node.builders.base_builder import measure_stage
-from build_node.build_node_errors import BuildError, BuildExcluded
+from build_node.models import Task
 from build_node.uploaders.pulp import PulpRpmUploader
+from build_node.utils.codenotary import notarize_build_artifacts
 from build_node.utils.file_utils import (
     clean_dir,
     filter_files,
     rm_sudo,
 )
-from build_node.models import Task
-from build_node.utils.codenotary import notarize_build_artifacts
 
 
 class BuildNodeBuilder(threading.Thread):

Bandit report

Run started:2024-04-04 14:43:39.561793

Test results:
>> Issue: [B311:blacklist] Standard pseudo-random generators are not suitable for security/cryptographic purposes.
   Severity: Low   Confidence: High
   CWE: CWE-330 (https://cwe.mitre.org/data/definitions/330.html)
   More Info: https://bandit.readthedocs.io/en/1.7.8/blacklists/blacklist_calls.html#b311-random
   Location: ./build_node/build_node_builder.py:105:45
104	                self.__logger.debug('there are no tasks to process')
105	                self.__terminated_event.wait(random.randint(5, 10))
106	                continue

--------------------------------------------------

Code scanned:
	Total lines of code: 407
	Total lines skipped (#nosec): 0
	Total potential issues skipped due to specifically being disabled (e.g., #nosec BXXX): 0

Run metrics:
	Total issues (by severity):
		Undefined: 0
		Low: 1
		Medium: 0
		High: 0
	Total issues (by confidence):
		Undefined: 0
		Low: 0
		Medium: 0
		High: 1
Files skipped (0):