From 2a6f112da775a13422a5274017789cac9c46b3ad Mon Sep 17 00:00:00 2001 From: Douglas Coburn Date: Fri, 9 Aug 2024 14:32:38 -0700 Subject: [PATCH 1/3] Added debug logic to find files --- .gitignore | 3 ++- pyproject.toml | 3 ++- requirements.txt | 3 ++- socketsecurity/__init__.py | 2 +- socketsecurity/core/__init__.py | 12 ++++++++++++ socketsecurity/socketcli.py | 5 +++-- 6 files changed, 22 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index a4d6257..5738fef 100644 --- a/.gitignore +++ b/.gitignore @@ -18,4 +18,5 @@ markdown_security_temp.md .DS_Store *.pyc test.py -*.cpython-312.pyc \ No newline at end of file +*.cpython-312.pyc` +file_generator.py \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 7568ed7..2c8ddf8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,7 +11,8 @@ dependencies = [ 'mdutils', 'prettytable', 'argparse', - 'GitPython' + 'GitPython', + 'packaging' ] readme = "README.md" description = "Socket Security CLI for CI/CD" diff --git a/requirements.txt b/requirements.txt index dfd906d..896774a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,4 +2,5 @@ requests>=2.32.0 mdutils~=1.6.0 prettytable argparse -gitpython>=3.1.43 \ No newline at end of file +gitpython>=3.1.43 +packaging>=24.1 \ No newline at end of file diff --git a/socketsecurity/__init__.py b/socketsecurity/__init__.py index 29709fd..c9cd5d7 100644 --- a/socketsecurity/__init__.py +++ b/socketsecurity/__init__.py @@ -1,2 +1,2 @@ __author__ = 'socket.dev' -__version__ = '1.0.7' +__version__ = '1.0.13' diff --git a/socketsecurity/core/__init__.py b/socketsecurity/core/__init__.py index 8a0e29a..55ed6e7 100644 --- a/socketsecurity/core/__init__.py +++ b/socketsecurity/core/__init__.py @@ -46,6 +46,8 @@ all_new_alerts = False security_policy = {} log = logging.getLogger("socketdev") +log_format = "%(asctime)s %(funcName)20s() %(message)s" +logging.basicConfig(format=log_format) log.addHandler(logging.NullHandler()) socket_globs = { @@ -398,6 +400,8 @@ def find_files(path: str, files: list = None) -> list: """ all_files = [] files_provided = False + log.debug("Starting Find Files") + start_time = time.time() if files is not None and len(files) > 0: files_provided = True for ecosystem in socket_globs: @@ -405,11 +409,15 @@ def find_files(path: str, files: list = None) -> list: for file_name in patterns: pattern = patterns[file_name]["pattern"] file_path = f"{path}/**/{pattern}" + if not files_provided: + log.debug(f"Globbing {file_path}") files = glob(file_path, recursive=True) else: + log.debug("Files found from commit") files = Core.match_supported_files(path, files) for file in files: + log.debug(f"Checking {file} for match") if platform.system() == "Windows": file = file.replace("\\", "/") if path not in file: @@ -418,6 +426,10 @@ def find_files(path: str, files: list = None) -> list: details = (found_path, file_name) if details not in all_files: all_files.append(details) + log.debug("Finished Find Files") + end_time = time.time() + total_time = end_time - start_time + log.info(f"Found {len(all_files)} in {total_time: 2f} seconds") return all_files @staticmethod diff --git a/socketsecurity/socketcli.py b/socketsecurity/socketcli.py index 722c587..2168222 100644 --- a/socketsecurity/socketcli.py +++ b/socketsecurity/socketcli.py @@ -211,7 +211,8 @@ def main_code(): arguments = parser.parse_args() debug = arguments.enable_debug if debug: - logging.basicConfig(level=logging.DEBUG) + log_format = "%(asctime)s %(funcName)20s() %(message)s" + logging.basicConfig(level=logging.DEBUG, format=log_format) log.setLevel(logging.DEBUG) Core.enable_debug_log(logging.DEBUG) log.debug("Debug logging enabled") @@ -287,7 +288,7 @@ def main_code(): default_branch = scm.is_default_branch base_api_url = os.getenv("BASE_API_URL") or None - core = Core(token=api_token, request_timeout=6000, base_api_url=base_api_url) + core = Core(token=api_token, request_timeout=1200, base_api_url=base_api_url) no_change = True if ignore_commit_files: no_change = False From 88730bba1a4f0408c0d6915c859af47c9d4d2f9c Mon Sep 17 00:00:00 2001 From: Douglas Coburn Date: Fri, 9 Aug 2024 15:32:54 -0700 Subject: [PATCH 2/3] Added more debug logging for find files --- socketsecurity/__init__.py | 2 +- socketsecurity/core/__init__.py | 21 +++++++++++++++------ socketsecurity/socketcli.py | 7 +++++-- 3 files changed, 21 insertions(+), 9 deletions(-) diff --git a/socketsecurity/__init__.py b/socketsecurity/__init__.py index c9cd5d7..884c377 100644 --- a/socketsecurity/__init__.py +++ b/socketsecurity/__init__.py @@ -1,2 +1,2 @@ __author__ = 'socket.dev' -__version__ = '1.0.13' +__version__ = '1.0.14' diff --git a/socketsecurity/core/__init__.py b/socketsecurity/core/__init__.py index 55ed6e7..688d6ba 100644 --- a/socketsecurity/core/__init__.py +++ b/socketsecurity/core/__init__.py @@ -46,8 +46,8 @@ all_new_alerts = False security_policy = {} log = logging.getLogger("socketdev") -log_format = "%(asctime)s %(funcName)20s() %(message)s" -logging.basicConfig(format=log_format) +# log_format = "%(asctime)s %(funcName)20s() %(message)s" +# logging.basicConfig(format=log_format) log.addHandler(logging.NullHandler()) socket_globs = { @@ -398,7 +398,7 @@ def find_files(path: str, files: list = None) -> list: :param files: override finding the manifest files using the glob matcher :return: """ - all_files = [] + all_files = set() files_provided = False log.debug("Starting Find Files") start_time = time.time() @@ -412,12 +412,17 @@ def find_files(path: str, files: list = None) -> list: if not files_provided: log.debug(f"Globbing {file_path}") + glob_start = time.time() files = glob(file_path, recursive=True) + glob_end = time.time() + glob_total_time = glob_end - glob_start + log.debug(f"Glob for pattern {file_path} took {glob_total_time:.2f} seconds") else: log.debug("Files found from commit") files = Core.match_supported_files(path, files) + name_fix_start = time.time() for file in files: - log.debug(f"Checking {file} for match") + # log.debug(f"Getting file and path for {file_path}") if platform.system() == "Windows": file = file.replace("\\", "/") if path not in file: @@ -425,11 +430,15 @@ def find_files(path: str, files: list = None) -> list: found_path, file_name = file.rsplit("/", 1) details = (found_path, file_name) if details not in all_files: - all_files.append(details) + all_files.add(details) + name_fix_end = time.time() + total_name_fix = name_fix_end - name_fix_start + log.debug(f"Total Time for name fix for {file_path} was {total_name_fix:.6f}") log.debug("Finished Find Files") end_time = time.time() total_time = end_time - start_time - log.info(f"Found {len(all_files)} in {total_time: 2f} seconds") + log.info(f"Found {len(all_files)} in {total_time:.2f} seconds") + all_files = list(all_files) return all_files @staticmethod diff --git a/socketsecurity/socketcli.py b/socketsecurity/socketcli.py index 2168222..8a3db52 100644 --- a/socketsecurity/socketcli.py +++ b/socketsecurity/socketcli.py @@ -1,5 +1,7 @@ import argparse import json + +import socketsecurity.core from socketsecurity.core import Core, __version__ from socketsecurity.core.classes import FullScanParams, Diff, Package, Issue from socketsecurity.core.messages import Messages @@ -10,7 +12,9 @@ import sys import logging -logging.basicConfig(level=logging.INFO) +log_format = "%(asctime)s: %(message)s" +logging.basicConfig(level=logging.INFO, format=log_format) +socketsecurity.core.log.setLevel(level=logging.INFO) log = logging.getLogger("socketcli") blocking_disabled = False @@ -211,7 +215,6 @@ def main_code(): arguments = parser.parse_args() debug = arguments.enable_debug if debug: - log_format = "%(asctime)s %(funcName)20s() %(message)s" logging.basicConfig(level=logging.DEBUG, format=log_format) log.setLevel(logging.DEBUG) Core.enable_debug_log(logging.DEBUG) From 990a41f14cb1530872a9303f761a7def63e7c1bf Mon Sep 17 00:00:00 2001 From: Douglas Coburn Date: Mon, 12 Aug 2024 07:26:40 -0700 Subject: [PATCH 3/3] Changes to the find_files function to reduce memory usage --- socketsecurity/__init__.py | 2 +- socketsecurity/core/__init__.py | 39 ++++++++++++++++----------------- 2 files changed, 20 insertions(+), 21 deletions(-) diff --git a/socketsecurity/__init__.py b/socketsecurity/__init__.py index 884c377..e239fae 100644 --- a/socketsecurity/__init__.py +++ b/socketsecurity/__init__.py @@ -1,2 +1,2 @@ __author__ = 'socket.dev' -__version__ = '1.0.14' +__version__ = '1.0.15' diff --git a/socketsecurity/core/__init__.py b/socketsecurity/core/__init__.py index 688d6ba..f24480f 100644 --- a/socketsecurity/core/__init__.py +++ b/socketsecurity/core/__init__.py @@ -398,13 +398,14 @@ def find_files(path: str, files: list = None) -> list: :param files: override finding the manifest files using the glob matcher :return: """ - all_files = set() files_provided = False log.debug("Starting Find Files") start_time = time.time() if files is not None and len(files) > 0: files_provided = True for ecosystem in socket_globs: + if files is None: + files = [] patterns = socket_globs[ecosystem] for file_name in patterns: pattern = patterns[file_name]["pattern"] @@ -413,33 +414,19 @@ def find_files(path: str, files: list = None) -> list: if not files_provided: log.debug(f"Globbing {file_path}") glob_start = time.time() - files = glob(file_path, recursive=True) + test = glob(file_path, recursive=True) + files = files + test glob_end = time.time() glob_total_time = glob_end - glob_start log.debug(f"Glob for pattern {file_path} took {glob_total_time:.2f} seconds") else: log.debug("Files found from commit") files = Core.match_supported_files(path, files) - name_fix_start = time.time() - for file in files: - # log.debug(f"Getting file and path for {file_path}") - if platform.system() == "Windows": - file = file.replace("\\", "/") - if path not in file: - file = f"{path}/{file}" - found_path, file_name = file.rsplit("/", 1) - details = (found_path, file_name) - if details not in all_files: - all_files.add(details) - name_fix_end = time.time() - total_name_fix = name_fix_end - name_fix_start - log.debug(f"Total Time for name fix for {file_path} was {total_name_fix:.6f}") log.debug("Finished Find Files") end_time = time.time() total_time = end_time - start_time - log.info(f"Found {len(all_files)} in {total_time:.2f} seconds") - all_files = list(all_files) - return all_files + log.info(f"Found {len(files)} in {total_time:.2f} seconds") + return files @staticmethod def create_full_scan(files: list, params: FullScanParams, workspace: str) -> FullScan: @@ -451,7 +438,16 @@ def create_full_scan(files: list, params: FullScanParams, workspace: str) -> Ful :return: """ send_files = [] - for path, name in files: + create_full_start = time.time() + log.debug("Creating new full scan") + for file in files: + if platform.system() == "Windows": + file = file.replace("\\", "/") + if "/" in file: + path, name = file.rsplit("/", 1) + else: + path = "." + name = file full_path = f"{path}/{name}" if full_path.startswith(workspace): key = full_path[len(workspace):] @@ -473,6 +469,9 @@ def create_full_scan(files: list, params: FullScanParams, workspace: str) -> Ful results = response.json() full_scan = FullScan(**results) full_scan.sbom_artifacts = Core.get_sbom_data(full_scan.id) + create_full_end = time.time() + total_time = create_full_end - create_full_start + log.debug(f"New Full Scan created in {total_time:.2f} seconds") return full_scan @staticmethod