From e3c46c7f1c0e004f0c5b053379b34e281961717a Mon Sep 17 00:00:00 2001 From: gotbadger Date: Mon, 6 Oct 2025 09:27:55 +0100 Subject: [PATCH 1/2] CM-53667: improve ignore logging --- cycode/cli/files_collector/walk_ignore.py | 19 ++++++++++++++++--- cycode/cli/utils/ignore_utils.py | 23 +++++++++++++++++------ 2 files changed, 33 insertions(+), 9 deletions(-) diff --git a/cycode/cli/files_collector/walk_ignore.py b/cycode/cli/files_collector/walk_ignore.py index f0e8edd6..5964ca25 100644 --- a/cycode/cli/files_collector/walk_ignore.py +++ b/cycode/cli/files_collector/walk_ignore.py @@ -1,9 +1,11 @@ import os from collections.abc import Generator, Iterable -from cycode.cli.logger import logger +from cycode.cli.logger import get_logger from cycode.cli.utils.ignore_utils import IgnoreFilterManager +logger = get_logger('File Ignore') + _SUPPORTED_IGNORE_PATTERN_FILES = { '.gitignore', '.cycodeignore', @@ -30,7 +32,7 @@ def _collect_top_level_ignore_files(path: str) -> list[str]: for ignore_file in _SUPPORTED_IGNORE_PATTERN_FILES: ignore_file_path = os.path.join(dir_path, ignore_file) if os.path.exists(ignore_file_path): - logger.debug('Apply top level ignore file: %s', ignore_file_path) + logger.debug('Reading top level ignore file: %s', ignore_file_path) ignore_files.append(ignore_file_path) return ignore_files @@ -41,4 +43,15 @@ def walk_ignore(path: str) -> Generator[tuple[str, list[str], list[str]], None, global_ignore_file_paths=_collect_top_level_ignore_files(path), global_patterns=_DEFAULT_GLOBAL_IGNORE_PATTERNS, ) - yield from ignore_filter_manager.walk() + + for dirpath, dirnames, filenames, ignored_dirnames, ignored_filenames in ignore_filter_manager.walk_with_ignored(): + rel_dirpath = '' if dirpath == path else os.path.relpath(dirpath, path) + display_dir = rel_dirpath or '.' + for kind, names in ( + ('directory', ignored_dirnames), + ('file', ignored_filenames), + ): + for name in names: + logger.debug('Skipping matched %s %s/%s', kind, display_dir, name) + + yield dirpath, dirnames, filenames diff --git a/cycode/cli/utils/ignore_utils.py b/cycode/cli/utils/ignore_utils.py index e8994e46..af9de3b0 100644 --- a/cycode/cli/utils/ignore_utils.py +++ b/cycode/cli/utils/ignore_utils.py @@ -388,19 +388,30 @@ def is_ignored(self, path: str) -> Optional[bool]: return matches[-1].is_exclude return None - def walk(self, **kwargs) -> Generator[tuple[str, list[str], list[str]], None, None]: - """Wrap os.walk() without ignored files and subdirectories and kwargs are passed to walk.""" + def walk_with_ignored( + self, **kwargs + ) -> Generator[tuple[str, list[str], list[str], list[str], list[str]], None, None]: + """Wrap os.walk() and also return lists of ignored directories and files. + + Yields tuples: (dirpath, included_dirnames, included_filenames, ignored_dirnames, ignored_filenames) + """ for dirpath, dirnames, filenames in os.walk(self.path, topdown=True, **kwargs): rel_dirpath = '' if dirpath == self.path else os.path.relpath(dirpath, self.path) + original_dirnames = list(dirnames) + included_dirnames = [d for d in original_dirnames if not self.is_ignored(os.path.join(rel_dirpath, d))] + # decrease recursion depth of os.walk() by ignoring subdirectories because of topdown=True # slicing ([:]) is mandatory to change dict in-place! - dirnames[:] = [d for d in dirnames if not self.is_ignored(os.path.join(rel_dirpath, d))] + dirnames[:] = included_dirnames + + ignored_dirnames = [d for d in original_dirnames if d not in included_dirnames] - # remove ignored files - filenames = [f for f in filenames if not self.is_ignored(os.path.join(rel_dirpath, f))] + original_filenames = list(filenames) + included_filenames = [f for f in original_filenames if not self.is_ignored(os.path.join(rel_dirpath, f))] + ignored_filenames = [f for f in original_filenames if f not in included_filenames] - yield dirpath, dirnames, filenames + yield dirpath, dirnames, included_filenames, ignored_dirnames, ignored_filenames @classmethod def build( From 67abc9cf97a2e9a8eac9631be63f6ebf4cb96c1e Mon Sep 17 00:00:00 2001 From: gotbadger Date: Mon, 6 Oct 2025 12:06:46 +0100 Subject: [PATCH 2/2] CM-53667: improved logging and process --- cycode/cli/files_collector/walk_ignore.py | 14 ++++++++------ cycode/cli/utils/ignore_utils.py | 20 ++++++++++++++------ 2 files changed, 22 insertions(+), 12 deletions(-) diff --git a/cycode/cli/files_collector/walk_ignore.py b/cycode/cli/files_collector/walk_ignore.py index 5964ca25..fb723109 100644 --- a/cycode/cli/files_collector/walk_ignore.py +++ b/cycode/cli/files_collector/walk_ignore.py @@ -4,7 +4,7 @@ from cycode.cli.logger import get_logger from cycode.cli.utils.ignore_utils import IgnoreFilterManager -logger = get_logger('File Ignore') +logger = get_logger('Ignores') _SUPPORTED_IGNORE_PATTERN_FILES = { '.gitignore', @@ -43,15 +43,17 @@ def walk_ignore(path: str) -> Generator[tuple[str, list[str], list[str]], None, global_ignore_file_paths=_collect_top_level_ignore_files(path), global_patterns=_DEFAULT_GLOBAL_IGNORE_PATTERNS, ) - for dirpath, dirnames, filenames, ignored_dirnames, ignored_filenames in ignore_filter_manager.walk_with_ignored(): rel_dirpath = '' if dirpath == path else os.path.relpath(dirpath, path) display_dir = rel_dirpath or '.' - for kind, names in ( - ('directory', ignored_dirnames), - ('file', ignored_filenames), + for is_dir, names in ( + (True, ignored_dirnames), + (False, ignored_filenames), ): for name in names: - logger.debug('Skipping matched %s %s/%s', kind, display_dir, name) + full_path = os.path.join(path, display_dir, name) + if is_dir: + full_path = os.path.join(full_path, '*') + logger.debug('Ignoring match %s', full_path) yield dirpath, dirnames, filenames diff --git a/cycode/cli/utils/ignore_utils.py b/cycode/cli/utils/ignore_utils.py index af9de3b0..98126658 100644 --- a/cycode/cli/utils/ignore_utils.py +++ b/cycode/cli/utils/ignore_utils.py @@ -399,17 +399,25 @@ def walk_with_ignored( rel_dirpath = '' if dirpath == self.path else os.path.relpath(dirpath, self.path) original_dirnames = list(dirnames) - included_dirnames = [d for d in original_dirnames if not self.is_ignored(os.path.join(rel_dirpath, d))] + included_dirnames = [] + ignored_dirnames = [] + for d in original_dirnames: + if self.is_ignored(os.path.join(rel_dirpath, d)): + ignored_dirnames.append(d) + else: + included_dirnames.append(d) # decrease recursion depth of os.walk() by ignoring subdirectories because of topdown=True # slicing ([:]) is mandatory to change dict in-place! dirnames[:] = included_dirnames - ignored_dirnames = [d for d in original_dirnames if d not in included_dirnames] - - original_filenames = list(filenames) - included_filenames = [f for f in original_filenames if not self.is_ignored(os.path.join(rel_dirpath, f))] - ignored_filenames = [f for f in original_filenames if f not in included_filenames] + included_filenames = [] + ignored_filenames = [] + for f in filenames: + if self.is_ignored(os.path.join(rel_dirpath, f)): + ignored_filenames.append(f) + else: + included_filenames.append(f) yield dirpath, dirnames, included_filenames, ignored_dirnames, ignored_filenames