feat(sca): Add a --support flag (bridgecrewio#4323)

* hi

* add --support flag to send customer logs to S3 and add extend data to the run metadata

* Revert "hi"

This reverts commit 0e1dc22.

* Revert "hi"

This reverts commit 0e1dc22.

* attach some changes

* fix main.py file

* make run metadata take string attribute

* make run metadata take string attribute

* add string atr to metadata into run metadata

* add string atr to metadata into run metadata

* add new changes

* reformat main.py

* add some changes main.py

* add some changes main.py

* add some changes to CLI

* add changes to clean up and finalize it

* add new two functions to sprite the run metadata and log stream handler + add tar compression

* remove print line

* create a new method into file_utils.py to compress stringIO + logging error to any exceptions caught

* support persisted errors

* change paser error to logging warning after all api key checks

---------

Co-authored-by: ReemShamasnah <remodo.1@hotmail.com>

checkov/common/bridgecrew/platform_integration.py

@@ -8,6 +8,7 @@
 import webbrowser
 from collections import namedtuple
 from concurrent import futures
+from io import StringIO
 from json import JSONDecodeError
 from os import path
 from pathlib import Path
@@ -30,7 +31,8 @@
 from checkov.common.bridgecrew.platform_errors import BridgecrewAuthError
 from checkov.common.bridgecrew.platform_key import read_key, persist_key, bridgecrew_file
 from checkov.common.bridgecrew.wrapper import reduce_scan_reports, persist_checks_results, \
-    enrich_and_persist_checks_metadata, checkov_results_prefix, persist_run_metadata, _put_json_object
+    enrich_and_persist_checks_metadata, checkov_results_prefix, persist_run_metadata, _put_json_object, \
+    persist_logs_stream
 from checkov.common.models.consts import SUPPORTED_FILE_EXTENSIONS, SUPPORTED_FILES, SCANNABLE_PACKAGE_FILES
 from checkov.common.bridgecrew.check_type import CheckType
 from checkov.common.runners.base_runner import filter_ignored_paths
@@ -452,6 +454,14 @@ def persist_run_metadata(self, run_metadata: dict[str, str | list[str]]) -> None
             return
         persist_run_metadata(run_metadata, self.s3_client, self.bucket, self.repo_path)
 
+    def persist_logs_stream(self, logs_stream: StringIO) -> None:
+        if not self.use_s3_integration:
+            return
+        if not self.bucket or not self.repo_path:
+            logging.error(f"Something went wrong: bucket {self.bucket}, repo path {self.repo_path}")
+            return
+        persist_logs_stream(logs_stream, self.s3_client, self.bucket, self.repo_path)
+
     def commit_repository(self, branch: str) -> str | None:
         """
         :param branch: branch to be persisted

checkov/common/bridgecrew/wrapper.py

@@ -4,6 +4,7 @@
 import os
 import json
 import itertools
+from io import StringIO
 from typing import Any, TYPE_CHECKING
 from collections import defaultdict
 
@@ -12,6 +13,7 @@
 from checkov.common.bridgecrew.check_type import CheckType
 from checkov.common.models.consts import SUPPORTED_FILE_EXTENSIONS
 from checkov.common.typing import _ReducedScanReport
+from checkov.common.util.file_utils import compress_string_io_tar
 from checkov.common.util.json_utils import CustomJSONEncoder
 
 if TYPE_CHECKING:
@@ -101,11 +103,22 @@ def persist_run_metadata(
     object_path = f'{full_repo_object_key}/{checkov_results_prefix}/run_metadata.json'
     try:
         s3_client.put_object(Bucket=bucket, Key=object_path, Body=json.dumps(run_metadata, indent=2))
+
     except Exception:
         logging.error(f"failed to persist run metadata into S3 bucket {bucket}", exc_info=True)
         raise
 
 
+def persist_logs_stream(logs_stream: StringIO, s3_client: BaseClient, bucket: str, full_repo_object_key: str) -> None:
+    file_io = compress_string_io_tar(logs_stream)
+    object_path = f'{full_repo_object_key}/{checkov_results_prefix}/logs_file.tar.gz'
+    try:
+        s3_client.put_object(Bucket=bucket, Key=object_path, Body=file_io)
+    except Exception:
+        logging.error(f"failed to persist logs stream into S3 bucket {bucket}", exc_info=True)
+        raise
+
+
 def enrich_and_persist_checks_metadata(
         scan_reports: list[Report], s3_client: BaseClient, bucket: str, full_repo_object_key: str
 ) -> dict[str, dict[str, str]]:

checkov/common/util/ext_argument_parser.py

@@ -115,6 +115,12 @@ def add_parser_args(self) -> None:
             action="version",
             version=version,
         )
+        self.add(
+            "--support",
+            action="store_true",
+            help="Enable debug logs and upload the logs to the server. Requires a Bridgecrew or Prisma Cloud API key.",
+            default=None
+        )
         self.add(
             "-d",
             "--directory",

checkov/common/util/file_utils.py

@@ -40,3 +40,20 @@ def decompress_file_gzip_base64(compressed_file_body: str) -> bytes:
     except Exception:
         logging.exception("failed to extract package file")
         raise
+
+
+def compress_string_io_tar(string_io: io.StringIO) -> io.BytesIO:
+    file_io = io.BytesIO()
+    str_data = string_io.getvalue().encode('utf8')
+    bio = io.BytesIO(str_data)
+    try:
+        with tarfile.open(fileobj=file_io, mode='w:gz') as tar:
+            info = tar.tarinfo(name='logs_file.txt')
+            bio.seek(0)
+            info.size = string_io.tell()
+            tar.addfile(info, bio)
+        file_io.seek(0)
+        return file_io
+    except Exception:
+        logging.exception("failed to compress logging file")
+        raise

checkov/logging_init.py

@@ -1,6 +1,6 @@
 import logging
 import os
-
+from io import StringIO
 
 LOG_LEVEL = os.environ.get('LOG_LEVEL', 'WARNING').upper()
 logging.basicConfig(level=LOG_LEVEL)
@@ -14,3 +14,8 @@
 logging.getLogger("urllib3.connectionpool").setLevel(logging.ERROR)
 logging.getLogger("urllib3.connectionpool").propagate = False
 logging.getLogger("urllib3").propagate = False
+log_stream = StringIO()
+stream_handler = logging.StreamHandler(stream=log_stream)
+stream_handler.setFormatter(log_formatter)
+stream_handler.setLevel(logging.DEBUG)
+root_logger.addHandler(stream_handler)