From 39ef96adf72eacfe679069f978f9f69527737d7d Mon Sep 17 00:00:00 2001 From: Chris Lee Date: Tue, 4 Feb 2025 18:19:02 -0800 Subject: [PATCH 01/17] wip: initial implementation of a codeowners interface --- src/codegen/git/utils/cache_utils.py | 43 ++++++++++++++++ src/codegen/sdk/core/codebase.py | 13 +++++ src/codegen/sdk/core/codeowner.py | 75 ++++++++++++++++++++++++++++ 3 files changed, 131 insertions(+) create mode 100644 src/codegen/git/utils/cache_utils.py create mode 100644 src/codegen/sdk/core/codeowner.py diff --git a/src/codegen/git/utils/cache_utils.py b/src/codegen/git/utils/cache_utils.py new file mode 100644 index 000000000..784bf0dc0 --- /dev/null +++ b/src/codegen/git/utils/cache_utils.py @@ -0,0 +1,43 @@ +import functools +from collections.abc import Iterator +from typing import Callable, Generic, ParamSpec, TypeVar + +ItemType = TypeVar("ItemType") +GenParamSpec = ParamSpec("GenParamSpec") + + +class LazyGeneratorCache(Generic[ItemType]): + """A cache for a generator that is lazily evaluated.""" + + _cache: list[ItemType] + gen: Iterator[ItemType] + + def __init__(self, gen: Iterator[ItemType]): + self._cache = [] + self.gen = gen + + def __iter__(self) -> Iterator[ItemType]: + for item in self._cache: + yield item + + for item in self.gen: + self._cache.append(item) + yield item + + +def cached_generator(maxsize: int = 16, typed: bool = False) -> Callable[[Callable[GenParamSpec, Iterator[ItemType]]], Callable[GenParamSpec, Iterator[ItemType]]]: + """Decorator to cache the output of a generator function. + + The generator's output is fully consumed on the first call and stored as a list. + Subsequent calls with the same arguments yield values from the cached list. + """ + + def decorator(func: Callable[GenParamSpec, Iterator[ItemType]]) -> Callable[GenParamSpec, Iterator[ItemType]]: + @functools.lru_cache(maxsize=maxsize, typed=typed) + @functools.wraps(func) + def wrapper(*args: GenParamSpec.args, **kwargs: GenParamSpec.kwargs) -> Iterator[ItemType]: + return LazyGeneratorCache(func(*args, **kwargs)) + + return wrapper + + return decorator diff --git a/src/codegen/sdk/core/codebase.py b/src/codegen/sdk/core/codebase.py index cee8f58a0..0cce5f36e 100644 --- a/src/codegen/sdk/core/codebase.py +++ b/src/codegen/sdk/core/codebase.py @@ -7,6 +7,7 @@ import re from collections.abc import Generator from contextlib import contextmanager +from functools import cached_property from pathlib import Path from typing import TYPE_CHECKING, Generic, Literal, TypeVar, Unpack, overload @@ -35,6 +36,7 @@ from codegen.sdk.codebase.span import Span from codegen.sdk.core.assignment import Assignment from codegen.sdk.core.class_definition import Class +from codegen.sdk.core.codeowner import CodeOwner from codegen.sdk.core.detached_symbols.code_block import CodeBlock from codegen.sdk.core.detached_symbols.parameter import Parameter from codegen.sdk.core.directory import Directory @@ -256,6 +258,17 @@ def files(self, *, extensions: list[str] | Literal["*"] | None = None) -> list[T # Sort files alphabetically return sort_editables(files, alphabetical=True, dedupe=False) + @cached_property + def codeowners(self) -> list["CodeOwner[TSourceFile]"]: + """List all CodeOnwers in the codebase. + + Returns: + list[CodeOwners]: A list of CodeOwners objects in the codebase. + """ + if self.G.codeowners_parser is None: + return [] + return CodeOwner.from_parser(self.G.codeowners_parser, lambda *args, **kwargs: self.files(*args, **kwargs)) + @property def directories(self) -> list[TDirectory]: """List all directories in the codebase. diff --git a/src/codegen/sdk/core/codeowner.py b/src/codegen/sdk/core/codeowner.py new file mode 100644 index 000000000..cf84fa16d --- /dev/null +++ b/src/codegen/sdk/core/codeowner.py @@ -0,0 +1,75 @@ +from collections.abc import Iterable, Iterator +from typing import TYPE_CHECKING, Callable, Generic, Literal, ParamSpec, TypeVar + +from codeowners import CodeOwners as CodeOwnersParser + +from codegen.git.utils.cache_utils import cached_generator +from codegen.shared.decorators.docs import apidoc + +if TYPE_CHECKING: + from codegen.sdk.core.file import SourceFile + +import logging + +logger = logging.getLogger(__name__) + + +TSourceFile = TypeVar("TSourceFile", bound="SourceFile") +SourceParam = ParamSpec("SourceParam") + + +@apidoc +class CodeOwner(Generic[TSourceFile]): + """CodeOwner is a class that represents a code owner in a codebase. + + It is used to iterate over all files that are owned by a specific owner. + + Attributes: + parser: The CodeOwnersParser that was used to parse the codeowners file. + owner_type: The type of the owner (USERNAME, TEAM, EMAIL). + owner_value: The value of the owner. + files_source: A callable that returns an iterable of all files in the codebase. + """ + + owner_type: Literal["USERNAME", "TEAM", "EMAIL"] + owner_value: str + files_source: Callable[SourceParam, Iterable[TSourceFile]] + + def __init__(self, files_source: Callable[SourceParam, Iterable[TSourceFile]], owner_type: Literal["USERNAME", "TEAM", "EMAIL"], owner_value: str): + self.owner_type = owner_type + self.owner_value = owner_value + self.files_source = files_source + + @classmethod + def from_parser(cls, parser: CodeOwnersParser, file_source: Callable[SourceParam, Iterable[TSourceFile]]) -> list["CodeOwner"]: + """Create a list of CodeOwner objects from a CodeOwnersParser. + + Args: + parser (CodeOwnersParser): The CodeOwnersParser to use. + file_source (Callable[SourceParam, Iterable[TSourceFile]]): A callable that returns an iterable of all files in the codebase. + + Returns: + list[CodeOwner]: A list of CodeOwner objects. + """ + codeowners = [] + for _, _, owners, _, _ in parser.paths: + for owner_label, owner_value in owners: + codeowners.append(CodeOwner(file_source, owner_label, owner_value)) + return codeowners + + @cached_generator(maxsize=16) + def files(self, *args: SourceParam.args, **kwargs: SourceParam.kwargs) -> Iterable[TSourceFile]: + for source_file in self.files_source(*args, **kwargs): + # Filter files by owner value + if self.owner_value in source_file.owners: + yield source_file + + @property + def name(self) -> str: + return self.owner_value + + def __iter__(self) -> Iterator[TSourceFile]: + return self.files() + + def __repr__(self) -> str: + return f"CodeOwner(owner_type={self.owner_type}, owner_value={self.owner_value})" From d6e2a27160fa24e81b0504c8738683fa03a6215f Mon Sep 17 00:00:00 2001 From: clee-codegen <185840274+clee-codegen@users.noreply.github.com> Date: Wed, 5 Feb 2025 02:31:04 +0000 Subject: [PATCH 02/17] Automated pre-commit update --- ruff.toml | 1 + src/codegen/shared/compilation/function_imports.py | 1 + 2 files changed, 2 insertions(+) diff --git a/ruff.toml b/ruff.toml index f30d38974..e638ba30c 100644 --- a/ruff.toml +++ b/ruff.toml @@ -57,6 +57,7 @@ extend-generics = [ "codegen.sdk.core.assignment.Assignment", "codegen.sdk.core.class_definition.Class", "codegen.sdk.core.codebase.Codebase", + "codegen.sdk.core.codeowner.CodeOwner", "codegen.sdk.core.dataclasses.usage.Usage", "codegen.sdk.core.dataclasses.usage.UsageType", "codegen.sdk.core.dataclasses.usage.UsageKind", diff --git a/src/codegen/shared/compilation/function_imports.py b/src/codegen/shared/compilation/function_imports.py index c020230b5..f8539926c 100644 --- a/src/codegen/shared/compilation/function_imports.py +++ b/src/codegen/shared/compilation/function_imports.py @@ -28,6 +28,7 @@ def get_generated_imports(): from codegen.sdk.core.codebase import CodebaseType from codegen.sdk.core.codebase import PyCodebaseType from codegen.sdk.core.codebase import TSCodebaseType +from codegen.sdk.core.codeowner import CodeOwner from codegen.sdk.core.dataclasses.usage import Usage from codegen.sdk.core.dataclasses.usage import UsageKind from codegen.sdk.core.dataclasses.usage import UsageType From 7e546c7e63d1f91be5c8b54bdd7ca53b11e78a20 Mon Sep 17 00:00:00 2001 From: Chris Lee Date: Wed, 5 Feb 2025 00:13:30 -0800 Subject: [PATCH 03/17] add: symbols property to codeowner --- src/codegen/sdk/core/codeowner.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/codegen/sdk/core/codeowner.py b/src/codegen/sdk/core/codeowner.py index cf84fa16d..36f789dea 100644 --- a/src/codegen/sdk/core/codeowner.py +++ b/src/codegen/sdk/core/codeowner.py @@ -1,4 +1,5 @@ from collections.abc import Iterable, Iterator +from itertools import chain from typing import TYPE_CHECKING, Callable, Generic, Literal, ParamSpec, TypeVar from codeowners import CodeOwners as CodeOwnersParser @@ -8,7 +9,7 @@ if TYPE_CHECKING: from codegen.sdk.core.file import SourceFile - + from codegen.sdk.core.symbol import Symbol import logging logger = logging.getLogger(__name__) @@ -16,6 +17,7 @@ TSourceFile = TypeVar("TSourceFile", bound="SourceFile") SourceParam = ParamSpec("SourceParam") +TSymbol = TypeVar("TSymbol", bound="Symbol") @apidoc @@ -64,6 +66,11 @@ def files(self, *args: SourceParam.args, **kwargs: SourceParam.kwargs) -> Iterab if self.owner_value in source_file.owners: yield source_file + @property + def symbols(self) -> list[TSymbol]: + """Get a recursive list of all symbols in the directory and its subdirectories.""" + return list(chain.from_iterable(f.symbols for f in self.files)) + @property def name(self) -> str: return self.owner_value From 2cf9a363614ba383c8af960daf54f7889150c66f Mon Sep 17 00:00:00 2001 From: Chris Lee Date: Wed, 5 Feb 2025 15:59:41 -0800 Subject: [PATCH 04/17] add: fuller directory API refactor --- src/codegen/sdk/core/codeowner.py | 32 ++--- src/codegen/sdk/core/directory.py | 113 ++++-------------- .../sdk/core/interfaces/files_interface.py | 113 ++++++++++++++++++ 3 files changed, 145 insertions(+), 113 deletions(-) create mode 100644 src/codegen/sdk/core/interfaces/files_interface.py diff --git a/src/codegen/sdk/core/codeowner.py b/src/codegen/sdk/core/codeowner.py index 36f789dea..c4f6c5202 100644 --- a/src/codegen/sdk/core/codeowner.py +++ b/src/codegen/sdk/core/codeowner.py @@ -1,33 +1,29 @@ +import logging from collections.abc import Iterable, Iterator -from itertools import chain -from typing import TYPE_CHECKING, Callable, Generic, Literal, ParamSpec, TypeVar +from typing import TYPE_CHECKING, Callable, Literal, TypeVar from codeowners import CodeOwners as CodeOwnersParser from codegen.git.utils.cache_utils import cached_generator +from codegen.sdk.core.interfaces.files_interface import FilesInterface, FilesParam, TFile from codegen.shared.decorators.docs import apidoc if TYPE_CHECKING: - from codegen.sdk.core.file import SourceFile from codegen.sdk.core.symbol import Symbol -import logging logger = logging.getLogger(__name__) -TSourceFile = TypeVar("TSourceFile", bound="SourceFile") -SourceParam = ParamSpec("SourceParam") TSymbol = TypeVar("TSymbol", bound="Symbol") @apidoc -class CodeOwner(Generic[TSourceFile]): +class CodeOwner(FilesInterface): """CodeOwner is a class that represents a code owner in a codebase. It is used to iterate over all files that are owned by a specific owner. Attributes: - parser: The CodeOwnersParser that was used to parse the codeowners file. owner_type: The type of the owner (USERNAME, TEAM, EMAIL). owner_value: The value of the owner. files_source: A callable that returns an iterable of all files in the codebase. @@ -35,20 +31,21 @@ class CodeOwner(Generic[TSourceFile]): owner_type: Literal["USERNAME", "TEAM", "EMAIL"] owner_value: str - files_source: Callable[SourceParam, Iterable[TSourceFile]] + files_source: Callable[FilesParam, Iterable[TFile]] - def __init__(self, files_source: Callable[SourceParam, Iterable[TSourceFile]], owner_type: Literal["USERNAME", "TEAM", "EMAIL"], owner_value: str): + def __init__(self, files_source: Callable[FilesParam, Iterable[TFile]], owner_type: Literal["USERNAME", "TEAM", "EMAIL"], owner_value: str): self.owner_type = owner_type self.owner_value = owner_value self.files_source = files_source + self.files = self.files_generator @classmethod - def from_parser(cls, parser: CodeOwnersParser, file_source: Callable[SourceParam, Iterable[TSourceFile]]) -> list["CodeOwner"]: + def from_parser(cls, parser: CodeOwnersParser, file_source: Callable[FilesParam, Iterable[TFile]]) -> list["CodeOwner"]: """Create a list of CodeOwner objects from a CodeOwnersParser. Args: parser (CodeOwnersParser): The CodeOwnersParser to use. - file_source (Callable[SourceParam, Iterable[TSourceFile]]): A callable that returns an iterable of all files in the codebase. + file_source (Callable[FilesParam, Iterable[TFile]]): A callable that returns an iterable of all files in the codebase. Returns: list[CodeOwner]: A list of CodeOwner objects. @@ -60,23 +57,18 @@ def from_parser(cls, parser: CodeOwnersParser, file_source: Callable[SourceParam return codeowners @cached_generator(maxsize=16) - def files(self, *args: SourceParam.args, **kwargs: SourceParam.kwargs) -> Iterable[TSourceFile]: + def files_generator(self, *args: FilesParam.args, **kwargs: FilesParam.kwargs) -> Iterable[TFile]: for source_file in self.files_source(*args, **kwargs): # Filter files by owner value if self.owner_value in source_file.owners: yield source_file - @property - def symbols(self) -> list[TSymbol]: - """Get a recursive list of all symbols in the directory and its subdirectories.""" - return list(chain.from_iterable(f.symbols for f in self.files)) - @property def name(self) -> str: return self.owner_value - def __iter__(self) -> Iterator[TSourceFile]: - return self.files() + def __iter__(self) -> Iterator[TFile]: + return self.files_generator() def __repr__(self) -> str: return f"CodeOwner(owner_type={self.owner_type}, owner_value={self.owner_value})" diff --git a/src/codegen/sdk/core/directory.py b/src/codegen/sdk/core/directory.py index fcbf8711a..2be589d70 100644 --- a/src/codegen/sdk/core/directory.py +++ b/src/codegen/sdk/core/directory.py @@ -1,17 +1,15 @@ +import logging import os -from itertools import chain +from collections.abc import Iterator from pathlib import Path -from typing import TYPE_CHECKING, Generic, Self, TypeVar +from typing import TYPE_CHECKING, Self, TypeVar +from codegen.git.utils.cache_utils import cached_generator +from codegen.sdk.core.interfaces.files_interface import FilesInterface, TFile from codegen.shared.decorators.docs import apidoc, py_noapidoc if TYPE_CHECKING: from codegen.sdk.core.assignment import Assignment - from codegen.sdk.core.class_definition import Class - from codegen.sdk.core.file import File - from codegen.sdk.core.function import Function - from codegen.sdk.core.import_resolution import Import, ImportStatement - from codegen.sdk.core.symbol import Symbol from codegen.sdk.typescript.class_definition import TSClass from codegen.sdk.typescript.export import TSExport from codegen.sdk.typescript.file import TSFile @@ -20,24 +18,13 @@ from codegen.sdk.typescript.statements.import_statement import TSImportStatement from codegen.sdk.typescript.symbol import TSSymbol -import logging +TSGlobalVar = TypeVar("TSGlobalVar", bound="Assignment") logger = logging.getLogger(__name__) -TFile = TypeVar("TFile", bound="File") -TSymbol = TypeVar("TSymbol", bound="Symbol") -TImportStatement = TypeVar("TImportStatement", bound="ImportStatement") -TGlobalVar = TypeVar("TGlobalVar", bound="Assignment") -TClass = TypeVar("TClass", bound="Class") -TFunction = TypeVar("TFunction", bound="Function") -TImport = TypeVar("TImport", bound="Import") - -TSGlobalVar = TypeVar("TSGlobalVar", bound="Assignment") - - @apidoc -class Directory(Generic[TFile, TSymbol, TImportStatement, TGlobalVar, TClass, TFunction, TImport]): +class Directory(FilesInterface): """Directory representation for codebase. GraphSitter abstraction of a file directory that can be used to look for files and symbols within a specific directory. @@ -58,7 +45,7 @@ def __init__(self, path: Path, dirpath: str, parent: Self | None): self.path = path self.dirpath = dirpath self.parent = parent - self.items = dict() + self.items = {} def __iter__(self): return iter(self.items.values()) @@ -126,62 +113,12 @@ def _get_subdirectories(directory: Directory): _get_subdirectories(self) return subdirectories - @property - def symbols(self) -> list[TSymbol]: - """Get a recursive list of all symbols in the directory and its subdirectories.""" - return list(chain.from_iterable(f.symbols for f in self.files)) - - @property - def import_statements(self) -> list[TImportStatement]: - """Get a recursive list of all import statements in the directory and its subdirectories.""" - return list(chain.from_iterable(f.import_statements for f in self.files)) - - @property - def global_vars(self) -> list[TGlobalVar]: - """Get a recursive list of all global variables in the directory and its subdirectories.""" - return list(chain.from_iterable(f.global_vars for f in self.files)) - - @property - def classes(self) -> list[TClass]: - """Get a recursive list of all classes in the directory and its subdirectories.""" - return list(chain.from_iterable(f.classes for f in self.files)) - - @property - def functions(self) -> list[TFunction]: - """Get a recursive list of all functions in the directory and its subdirectories.""" - return list(chain.from_iterable(f.functions for f in self.files)) - - @property - @py_noapidoc - def exports(self: "Directory[TSFile, TSSymbol, TSImportStatement, TSGlobalVar, TSClass, TSFunction, TSImport]") -> "list[TSExport]": - """Get a recursive list of all exports in the directory and its subdirectories.""" - return list(chain.from_iterable(f.exports for f in self.files)) - - @property - def imports(self) -> list[TImport]: - """Get a recursive list of all imports in the directory and its subdirectories.""" - return list(chain.from_iterable(f.imports for f in self.files)) - - def get_symbol(self, name: str) -> TSymbol | None: - """Get a symbol by name in the directory and its subdirectories.""" - return next((s for s in self.symbols if s.name == name), None) - - def get_import_statement(self, name: str) -> TImportStatement | None: - """Get an import statement by name in the directory and its subdirectories.""" - return next((s for s in self.import_statements if s.name == name), None) - - def get_global_var(self, name: str) -> TGlobalVar | None: - """Get a global variable by name in the directory and its subdirectories.""" - return next((s for s in self.global_vars if s.name == name), None) - - def get_class(self, name: str) -> TClass | None: - """Get a class by name in the directory and its subdirectories.""" - return next((s for s in self.classes if s.name == name), None) - - def get_function(self, name: str) -> TFunction | None: - """Get a function by name in the directory and its subdirectories.""" - return next((s for s in self.functions if s.name == name), None) + @cached_generator() + def files_generator(self) -> Iterator[TFile]: + """Yield files recursively from the directory.""" + yield from self.files + # Directory-specific methods def add_file(self, file: TFile) -> None: """Add a file to the directory.""" rel_path = os.path.relpath(file.file_path, self.dirpath) @@ -197,6 +134,11 @@ def remove_file_by_path(self, file_path: os.PathLike) -> None: rel_path = str(Path(file_path).relative_to(self.dirpath)) del self.items[rel_path] + @py_noapidoc + def get_export(self: "Directory[TSFile, TSSymbol, TSImportStatement, TSGlobalVar, TSClass, TSFunction, TSImport]", name: str) -> "TSExport | None": + """Get an export by name in the directory and its subdirectories (supports only typescript).""" + return next((s for s in self.exports if s.name == name), None) + def get_file(self, filename: str, ignore_case: bool = False) -> TFile | None: """Get a file by its name relative to the directory.""" from codegen.sdk.core.file import File @@ -205,15 +147,6 @@ def get_file(self, filename: str, ignore_case: bool = False) -> TFile | None: return next((f for name, f in self.items.items() if name.lower() == filename.lower() and isinstance(f, File)), None) return self.items.get(filename, None) - @py_noapidoc - def get_export(self: "Directory[TSFile, TSSymbol, TSImportStatement, TSGlobalVar, TSClass, TSFunction, TSImport]", name: str) -> "TSExport | None": - """Get an export by name in the directory and its subdirectories (supports only typescript).""" - return next((s for s in self.exports if s.name == name), None) - - def get_import(self, name: str) -> TImport | None: - """Get an import by name in the directory and its subdirectories.""" - return next((s for s in self.imports if s.name == name), None) - def add_subdirectory(self, subdirectory: Self) -> None: """Add a subdirectory to the directory.""" rel_path = os.path.relpath(subdirectory.dirpath, self.dirpath) @@ -230,19 +163,13 @@ def remove_subdirectory_by_path(self, subdirectory_path: str) -> None: del self.items[rel_path] def get_subdirectory(self, subdirectory_name: str) -> Self | None: - """Get a subdirectory by its path relative to the directory.""" + """Get a subdirectory by its name (relative to the directory).""" return self.items.get(subdirectory_name, None) - def remove(self) -> None: - """Remove the directory and all its files and subdirectories.""" - for f in self.files: - f.remove() - def update_filepath(self, new_filepath: str) -> None: - """Update the filepath of the directory.""" + """Update the filepath of the directory and its contained files.""" old_path = self.dirpath new_path = new_filepath - for file in self.files: new_file_path = os.path.join(new_path, os.path.relpath(file.file_path, old_path)) file.update_filepath(new_file_path) diff --git a/src/codegen/sdk/core/interfaces/files_interface.py b/src/codegen/sdk/core/interfaces/files_interface.py new file mode 100644 index 000000000..1a77e5f6d --- /dev/null +++ b/src/codegen/sdk/core/interfaces/files_interface.py @@ -0,0 +1,113 @@ +import logging +from collections.abc import Iterator +from itertools import chain +from typing import TYPE_CHECKING, Generic, ParamSpec, TypeVar + +from codegen.git.utils.cache_utils import cached_generator +from codegen.shared.decorators.docs import py_noapidoc + +if TYPE_CHECKING: + from codegen.sdk.core.assignment import Assignment + from codegen.sdk.core.class_definition import Class + from codegen.sdk.core.file import SourceFile + from codegen.sdk.core.function import Function + from codegen.sdk.core.import_resolution import Import, ImportStatement + from codegen.sdk.core.symbol import Symbol + from codegen.sdk.typescript.export import TSExport + + +logger = logging.getLogger(__name__) + + +TFile = TypeVar("TFile", bound="SourceFile") +TSymbol = TypeVar("TSymbol", bound="Symbol") +TImportStatement = TypeVar("TImportStatement", bound="ImportStatement") +TGlobalVar = TypeVar("TGlobalVar", bound="Assignment") +TClass = TypeVar("TClass", bound="Class") +TFunction = TypeVar("TFunction", bound="Function") +TImport = TypeVar("TImport", bound="Import") +FilesParam = ParamSpec("FilesParam") + +TSGlobalVar = TypeVar("TSGlobalVar", bound="Assignment") + + +class FilesInterface(Generic[TFile, TSymbol, TImportStatement, TGlobalVar, TClass, TFunction, TImport]): + """Abstract interface for files in a codebase. + + Abstract interface for files in a codebase. + """ + + @cached_generator() + def files_generator(self, *args: FilesParam.args, **kwargs: FilesParam.kwargs) -> Iterator[TFile]: + msg = "This method should be implemented by the subclass" + raise NotImplementedError(msg) + + @property + def symbols(self) -> list[TSymbol]: + """Get a recursive list of all symbols in files container.""" + return list(chain.from_iterable(f.symbols for f in self.files_generator())) + + @property + def import_statements(self) -> list[TImportStatement]: + """Get a recursive list of all import statements in files container.""" + return list(chain.from_iterable(f.import_statements for f in self.files_generator())) + + @property + def global_vars(self) -> list[TGlobalVar]: + """Get a recursive list of all global variables in files container.""" + return list(chain.from_iterable(f.global_vars for f in self.files_generator())) + + @property + def classes(self) -> list[TClass]: + """Get a recursive list of all classes in files container.""" + return list(chain.from_iterable(f.classes for f in self.files_generator())) + + @property + def functions(self) -> list[TFunction]: + """Get a recursive list of all functions in files container.""" + return list(chain.from_iterable(f.functions for f in self.files_generator())) + + @property + @py_noapidoc + def exports(self) -> "list[TSExport]": + """Get a recursive list of all exports in files container.""" + return list(chain.from_iterable(f.exports for f in self.files_generator())) + + @property + def imports(self) -> list[TImport]: + """Get a recursive list of all imports in files container.""" + return list(chain.from_iterable(f.imports for f in self.files_generator())) + + def get_symbol(self, name: str) -> TSymbol | None: + """Get a symbol by name in files container.""" + return next((s for s in self.symbols if s.name == name), None) + + def get_import_statement(self, name: str) -> TImportStatement | None: + """Get an import statement by name in files container.""" + return next((s for s in self.import_statements if s.name == name), None) + + def get_global_var(self, name: str) -> TGlobalVar | None: + """Get a global variable by name in files container.""" + return next((s for s in self.global_vars if s.name == name), None) + + def get_class(self, name: str) -> TClass | None: + """Get a class by name in files container.""" + return next((s for s in self.classes if s.name == name), None) + + def get_function(self, name: str) -> TFunction | None: + """Get a function by name in files container.""" + return next((s for s in self.functions if s.name == name), None) + + @py_noapidoc + def get_export(self, name: str) -> "TSExport | None": + """Get an export by name in files container (supports only typescript).""" + return next((s for s in self.exports if s.name == name), None) + + def get_import(self, name: str) -> TImport | None: + """Get an import by name in files container.""" + return next((s for s in self.imports if s.name == name), None) + + def remove(self) -> None: + """Remove all the files in the files container.""" + for f in self.files: + f.remove() From 9107c34ef80bbf4ff91dde410a590912a912d8d9 Mon Sep 17 00:00:00 2001 From: Chris Lee Date: Wed, 5 Feb 2025 16:02:39 -0800 Subject: [PATCH 05/17] fix: move remove to directory specific impl. --- src/codegen/sdk/core/codeowner.py | 12 ++------ src/codegen/sdk/core/directory.py | 30 ++++++------------- .../sdk/core/interfaces/files_interface.py | 14 ++++----- 3 files changed, 19 insertions(+), 37 deletions(-) diff --git a/src/codegen/sdk/core/codeowner.py b/src/codegen/sdk/core/codeowner.py index c4f6c5202..5af23140e 100644 --- a/src/codegen/sdk/core/codeowner.py +++ b/src/codegen/sdk/core/codeowner.py @@ -1,24 +1,18 @@ import logging from collections.abc import Iterable, Iterator -from typing import TYPE_CHECKING, Callable, Literal, TypeVar +from typing import Callable, Generic, Literal from codeowners import CodeOwners as CodeOwnersParser from codegen.git.utils.cache_utils import cached_generator -from codegen.sdk.core.interfaces.files_interface import FilesInterface, FilesParam, TFile +from codegen.sdk.core.interfaces.files_interface import FilesInterface, FilesParam, TClass, TFile, TFunction, TGlobalVar, TImport, TImportStatement, TSymbol from codegen.shared.decorators.docs import apidoc -if TYPE_CHECKING: - from codegen.sdk.core.symbol import Symbol - logger = logging.getLogger(__name__) -TSymbol = TypeVar("TSymbol", bound="Symbol") - - @apidoc -class CodeOwner(FilesInterface): +class CodeOwner(FilesInterface[TFile, TSymbol, TImportStatement, TGlobalVar, TClass, TFunction, TImport], Generic[TFile, TSymbol, TImportStatement, TGlobalVar, TClass, TFunction, TImport]): """CodeOwner is a class that represents a code owner in a codebase. It is used to iterate over all files that are owned by a specific owner. diff --git a/src/codegen/sdk/core/directory.py b/src/codegen/sdk/core/directory.py index 2be589d70..c0670adcb 100644 --- a/src/codegen/sdk/core/directory.py +++ b/src/codegen/sdk/core/directory.py @@ -2,29 +2,17 @@ import os from collections.abc import Iterator from pathlib import Path -from typing import TYPE_CHECKING, Self, TypeVar +from typing import Generic, Self from codegen.git.utils.cache_utils import cached_generator -from codegen.sdk.core.interfaces.files_interface import FilesInterface, TFile -from codegen.shared.decorators.docs import apidoc, py_noapidoc - -if TYPE_CHECKING: - from codegen.sdk.core.assignment import Assignment - from codegen.sdk.typescript.class_definition import TSClass - from codegen.sdk.typescript.export import TSExport - from codegen.sdk.typescript.file import TSFile - from codegen.sdk.typescript.function import TSFunction - from codegen.sdk.typescript.import_resolution import TSImport - from codegen.sdk.typescript.statements.import_statement import TSImportStatement - from codegen.sdk.typescript.symbol import TSSymbol - -TSGlobalVar = TypeVar("TSGlobalVar", bound="Assignment") +from codegen.sdk.core.interfaces.files_interface import FilesInterface, TClass, TFile, TFunction, TGlobalVar, TImport, TImportStatement, TSymbol +from codegen.shared.decorators.docs import apidoc logger = logging.getLogger(__name__) @apidoc -class Directory(FilesInterface): +class Directory(FilesInterface[TFile, TSymbol, TImportStatement, TGlobalVar, TClass, TFunction, TImport], Generic[TFile, TSymbol, TImportStatement, TGlobalVar, TClass, TFunction, TImport]): """Directory representation for codebase. GraphSitter abstraction of a file directory that can be used to look for files and symbols within a specific directory. @@ -134,11 +122,6 @@ def remove_file_by_path(self, file_path: os.PathLike) -> None: rel_path = str(Path(file_path).relative_to(self.dirpath)) del self.items[rel_path] - @py_noapidoc - def get_export(self: "Directory[TSFile, TSSymbol, TSImportStatement, TSGlobalVar, TSClass, TSFunction, TSImport]", name: str) -> "TSExport | None": - """Get an export by name in the directory and its subdirectories (supports only typescript).""" - return next((s for s in self.exports if s.name == name), None) - def get_file(self, filename: str, ignore_case: bool = False) -> TFile | None: """Get a file by its name relative to the directory.""" from codegen.sdk.core.file import File @@ -174,6 +157,11 @@ def update_filepath(self, new_filepath: str) -> None: new_file_path = os.path.join(new_path, os.path.relpath(file.file_path, old_path)) file.update_filepath(new_file_path) + def remove(self) -> None: + """Remove all the files in the files container.""" + for f in self.files: + f.remove() + def rename(self, new_name: str) -> None: """Rename the directory.""" parent_dir, _ = os.path.split(self.dirpath) diff --git a/src/codegen/sdk/core/interfaces/files_interface.py b/src/codegen/sdk/core/interfaces/files_interface.py index 1a77e5f6d..71e0be29e 100644 --- a/src/codegen/sdk/core/interfaces/files_interface.py +++ b/src/codegen/sdk/core/interfaces/files_interface.py @@ -13,8 +13,13 @@ from codegen.sdk.core.function import Function from codegen.sdk.core.import_resolution import Import, ImportStatement from codegen.sdk.core.symbol import Symbol + from codegen.sdk.typescript.class_definition import TSClass from codegen.sdk.typescript.export import TSExport - + from codegen.sdk.typescript.file import TSFile + from codegen.sdk.typescript.function import TSFunction + from codegen.sdk.typescript.import_resolution import TSImport + from codegen.sdk.typescript.statements.import_statement import TSImportStatement + from codegen.sdk.typescript.symbol import TSSymbol logger = logging.getLogger(__name__) @@ -99,15 +104,10 @@ def get_function(self, name: str) -> TFunction | None: return next((s for s in self.functions if s.name == name), None) @py_noapidoc - def get_export(self, name: str) -> "TSExport | None": + def get_export(self: "FilesInterface[TSFile, TSSymbol, TSImportStatement, TSGlobalVar, TSClass, TSFunction, TSImport]", name: str) -> "TSExport | None": """Get an export by name in files container (supports only typescript).""" return next((s for s in self.exports if s.name == name), None) def get_import(self, name: str) -> TImport | None: """Get an import by name in files container.""" return next((s for s in self.imports if s.name == name), None) - - def remove(self) -> None: - """Remove all the files in the files container.""" - for f in self.files: - f.remove() From 3bf30530a3647615a3cd96f262cb5b0d78be6ffe Mon Sep 17 00:00:00 2001 From: Chris Lee Date: Thu, 6 Feb 2025 16:42:00 -0800 Subject: [PATCH 06/17] add: unit tests for directory/interface/codeowner --- src/codegen/sdk/core/codeowner.py | 2 +- .../core/interfaces/test_files_interface.py | 153 ++++++++++++ tests/unit/codegen/sdk/core/test_codeowner.py | 71 ++++++ tests/unit/codegen/sdk/core/test_directory.py | 221 ++++++++++++++++++ 4 files changed, 446 insertions(+), 1 deletion(-) create mode 100644 tests/unit/codegen/sdk/core/interfaces/test_files_interface.py create mode 100644 tests/unit/codegen/sdk/core/test_codeowner.py create mode 100644 tests/unit/codegen/sdk/core/test_directory.py diff --git a/src/codegen/sdk/core/codeowner.py b/src/codegen/sdk/core/codeowner.py index 5af23140e..f494654ac 100644 --- a/src/codegen/sdk/core/codeowner.py +++ b/src/codegen/sdk/core/codeowner.py @@ -62,7 +62,7 @@ def name(self) -> str: return self.owner_value def __iter__(self) -> Iterator[TFile]: - return self.files_generator() + return iter(self.files_generator()) def __repr__(self) -> str: return f"CodeOwner(owner_type={self.owner_type}, owner_value={self.owner_value})" diff --git a/tests/unit/codegen/sdk/core/interfaces/test_files_interface.py b/tests/unit/codegen/sdk/core/interfaces/test_files_interface.py new file mode 100644 index 000000000..b8817af80 --- /dev/null +++ b/tests/unit/codegen/sdk/core/interfaces/test_files_interface.py @@ -0,0 +1,153 @@ +from unittest.mock import MagicMock + +import pytest + +from codegen.sdk.core.interfaces.files_interface import FilesInterface + + +@pytest.fixture +def fake_interface(): + class FakeFilesInterface(FilesInterface): + def __init__(self, files): + self._files = files + + def files_generator(self, *args, **kwargs): + yield from self._files + + # File 1 with its fake attributes. + file1 = MagicMock() + file1.symbols = [MagicMock(), MagicMock()] + file1.symbols[0].name = "symbol1" + file1.symbols[1].name = "symbol2" + file1.import_statements = [MagicMock()] + file1.import_statements[0].name = "import_statement1" + file1.global_vars = [MagicMock()] + file1.global_vars[0].name = "global_variable1" + file1.classes = [MagicMock()] + file1.classes[0].name = "class1" + file1.functions = [MagicMock()] + file1.functions[0].name = "function1" + file1.exports = [MagicMock()] + file1.exports[0].name = "export_item1" + file1.imports = [MagicMock()] + file1.imports[0].name = "import1" + + # File 2 with its fake attributes. + file2 = MagicMock() + file2.symbols = [MagicMock()] + file2.symbols[0].name = "symbol3" + file2.import_statements = [MagicMock()] + file2.import_statements[0].name = "import_statement2" + file2.global_vars = [MagicMock(), MagicMock()] + file2.global_vars[0].name = "global_variable2" + file2.global_vars[1].name = "global_variable3" + file2.classes = [MagicMock()] + file2.classes[0].name = "class2" + file2.functions = [MagicMock()] + file2.functions[0].name = "function2" + file2.exports = [MagicMock(), MagicMock()] + file2.exports[0].name = "export_item2" + file2.exports[1].name = "export_item3" + file2.imports = [MagicMock()] + file2.imports[0].name = "import2" + + fake_files = [file1, file2] + return FakeFilesInterface(fake_files) + + +def test_files_generator_not_implemented(): + # Instantiating FilesInterface directly should cause files_generator to raise NotImplementedError. + fi = FilesInterface() + with pytest.raises(NotImplementedError): + list(fi.files_generator()) + + +def test_symbols_property(fake_interface): + symbols = fake_interface.symbols + names = sorted([item.name for item in symbols]) + assert names == ["symbol1", "symbol2", "symbol3"] + + +def test_import_statements_property(fake_interface): + import_statements = fake_interface.import_statements + names = sorted([item.name for item in import_statements]) + assert names == ["import_statement1", "import_statement2"] + + +def test_global_vars_property(fake_interface): + global_vars = fake_interface.global_vars + names = sorted([item.name for item in global_vars]) + assert names == ["global_variable1", "global_variable2", "global_variable3"] + + +def test_classes_property(fake_interface): + classes = fake_interface.classes + names = sorted([item.name for item in classes]) + assert names == ["class1", "class2"] + + +def test_functions_property(fake_interface): + functions = fake_interface.functions + names = sorted([item.name for item in functions]) + assert names == ["function1", "function2"] + + +def test_exports_property(fake_interface): + exports = fake_interface.exports + names = sorted([item.name for item in exports]) + assert names == ["export_item1", "export_item2", "export_item3"] + + +def test_imports_property(fake_interface): + imports = fake_interface.imports + names = sorted([item.name for item in imports]) + assert names == ["import1", "import2"] + + +def test_get_symbol(fake_interface): + symbol = fake_interface.get_symbol("symbol1") + assert symbol is not None + assert symbol.name == "symbol1" + assert fake_interface.get_symbol("nonexistent") is None + + +def test_get_import_statement(fake_interface): + imp_stmt = fake_interface.get_import_statement("import_statement2") + assert imp_stmt is not None + assert imp_stmt.name == "import_statement2" + assert fake_interface.get_import_statement("nonexistent") is None + + +def test_get_global_var(fake_interface): + global_var = fake_interface.get_global_var("global_variable3") + assert global_var is not None + assert global_var.name == "global_variable3" + assert fake_interface.get_global_var("nonexistent") is None + + +def test_get_class(fake_interface): + cls = fake_interface.get_class("class2") + assert cls is not None + assert cls.name == "class2" + assert fake_interface.get_class("nonexistent") is None + + +def test_get_function(fake_interface): + func = fake_interface.get_function("function2") + assert func is not None + assert func.name == "function2" + assert fake_interface.get_function("nonexistent") is None + + +def test_get_export(fake_interface): + export_item = fake_interface.get_export("export_item3") + assert export_item is not None + assert export_item.name == "export_item3" + assert fake_interface.get_export("nonexistent") is None + + +def test_get_import(fake_interface): + imp = fake_interface.get_import("import1") + assert imp is not None + assert imp.name == "import1" + assert fake_interface.get_import("nonexistent") is None diff --git a/tests/unit/codegen/sdk/core/test_codeowner.py b/tests/unit/codegen/sdk/core/test_codeowner.py new file mode 100644 index 000000000..fc740bd47 --- /dev/null +++ b/tests/unit/codegen/sdk/core/test_codeowner.py @@ -0,0 +1,71 @@ +from collections.abc import Iterable +from unittest.mock import MagicMock + +import pytest + +from codegen.sdk.core.codeowner import CodeOwner + + +# Dummy file objects used for testing CodeOwner. +@pytest.fixture +def fake_files() -> Iterable[MagicMock]: + file1 = MagicMock() + file1.owners = ["alice", "bob"] + + file2 = MagicMock() + file2.owners = ["charlie"] + + file3 = MagicMock() + file3.owners = ["alice"] + + return [file1, file2, file3] + + +def test_files_generator_returns_correct_files(fake_files): + def file_source(*args, **kwargs): + return fake_files + + codeowner = CodeOwner(file_source, "USERNAME", "alice") + files = list(codeowner.files_generator()) + # file1 and file3 contain "alice" as one of their owners. + assert fake_files[0] in files + assert fake_files[2] in files + assert fake_files[1] not in files + + +def test_name_property_and_repr(): + def dummy_source(*args, **kwargs): + return [] + + codeowner = CodeOwner(dummy_source, "TEAM", "dev_team") + assert codeowner.name == "dev_team" + rep = repr(codeowner) + assert "TEAM" in rep and "dev_team" in rep + + +def test_iter_method(fake_files): + def file_source(*args, **kwargs): + return fake_files + + codeowner = CodeOwner(file_source, "USERNAME", "charlie") + iterated_files = list(codeowner) + assert iterated_files == [fake_files[1]] + + +def test_from_parser_method(fake_files): + # Create a fake parser with a paths attribute. + fake_parser = MagicMock() + fake_parser.paths = [ + ("pattern1", "ignored", [("USERNAME", "alice"), ("TEAM", "devs")], "ignored", "ignored"), + ("pattern2", "ignored", [("EMAIL", "bob@example.com")], "ignored", "ignored"), + ] + + def file_source(*args, **kwargs): + return fake_files + + codeowners = CodeOwner.from_parser(fake_parser, file_source) + assert len(codeowners) == 3 + owner_values = [co.owner_value for co in codeowners] + assert "alice" in owner_values + assert "devs" in owner_values + assert "bob@example.com" in owner_values diff --git a/tests/unit/codegen/sdk/core/test_directory.py b/tests/unit/codegen/sdk/core/test_directory.py new file mode 100644 index 000000000..1031730f2 --- /dev/null +++ b/tests/unit/codegen/sdk/core/test_directory.py @@ -0,0 +1,221 @@ +import os +import types +from pathlib import Path +from unittest.mock import MagicMock + +import pytest + +from codegen.sdk.codebase.codebase_graph import CodebaseGraph +from codegen.sdk.codebase.config import CodebaseConfig +from codegen.sdk.core.directory import Directory +from codegen.sdk.core.file import File + + +@pytest.fixture +def mock_codebase_graph(tmp_path): + mock = MagicMock(spec=CodebaseGraph) + mock.transaction_manager = MagicMock() + mock.config = CodebaseConfig() + mock.repo_path = tmp_path + mock.to_absolute = types.MethodType(CodebaseGraph.to_absolute, mock) + mock.to_relative = types.MethodType(CodebaseGraph.to_relative, mock) + return mock + + +@pytest.fixture +def subdir_path(tmp_path): + return tmp_path / "mock_dir" / "subdir" + + +@pytest.fixture +def dir_path(tmp_path): + return tmp_path / "mock_dir" + + +@pytest.fixture +def sub_dir(subdir_path, tmp_path): + return Directory(path=subdir_path.absolute(), dirpath=subdir_path.relative_to(tmp_path), parent=None) + + +@pytest.fixture +def mock_file(dir_path, mock_codebase_graph): + return File(filepath=dir_path / "example.py", G=mock_codebase_graph) + + +@pytest.fixture +def mock_directory(tmp_path, dir_path, sub_dir, mock_file): + directory = Directory(path=dir_path.absolute(), dirpath=dir_path.relative_to(tmp_path), parent=None) + directory.add_file(mock_file) + directory.add_subdirectory(sub_dir) + return directory + + +def test_directory_init(tmp_path, mock_directory): + """Test initialization of Directory object.""" + assert mock_directory.path == tmp_path / "mock_dir" + assert mock_directory.dirpath == Path("mock_dir") + assert mock_directory.parent is None + assert len(mock_directory.items) == 2 + assert mock_directory.items["subdir"] is not None + assert mock_directory.items["example.py"] is not None + + +def test_name_property(mock_directory): + """Test name property returns the basename of the dirpath.""" + assert mock_directory.name == "mock_dir" + + +def test_add_and_file(mock_directory, mock_codebase_graph): + """Test adding a file to the directory.""" + mock_file = File(filepath=Path("mock_dir/example_2.py"), G=mock_codebase_graph) + mock_directory.add_file(mock_file) + rel_path = os.path.relpath(mock_file.file_path, mock_directory.dirpath) + assert rel_path in mock_directory.items + assert mock_directory.items[rel_path] is mock_file + + +def test_remove_file(mock_directory, mock_file): + """Test removing a file from the directory.""" + mock_directory.remove_file(mock_file) + + rel_path = os.path.relpath(mock_file.file_path, mock_directory.dirpath) + assert rel_path not in mock_directory.items + + +def test_remove_file_by_path(mock_directory, mock_file): + """Test removing a file by path.""" + mock_directory.remove_file_by_path(mock_file.file_path) + + rel_path = os.path.relpath(mock_file.file_path, mock_directory.dirpath) + assert rel_path not in mock_directory.items + + +def test_get_file(mock_directory, mock_file): + """Test retrieving a file by name.""" + retrieved_file = mock_directory.get_file("example.py") + assert retrieved_file is mock_file + + # Case-insensitive match + retrieved_file_ci = mock_directory.get_file("EXAMPLE.PY", ignore_case=True) + assert retrieved_file_ci is mock_file + + +def test_get_file_not_found(mock_directory): + """Test retrieving a non-existing file returns None.""" + assert mock_directory.get_file("nonexistent.py") is None + + +def test_add_subdirectory(mock_directory, dir_path): + """Test adding a subdirectory.""" + new_subdir_path = dir_path / "new_subdir" + subdir = Directory(path=new_subdir_path.absolute(), dirpath=new_subdir_path.relative_to(dir_path), parent=mock_directory) + mock_directory.add_subdirectory(subdir) + rel_path = os.path.relpath(subdir.dirpath, mock_directory.dirpath) + assert rel_path in mock_directory.items + assert mock_directory.items[rel_path] is subdir + + +def test_remove_subdirectory(mock_directory, sub_dir): + """Test removing a subdirectory.""" + mock_directory.add_subdirectory(sub_dir) + mock_directory.remove_subdirectory(sub_dir) + + rel_path = os.path.relpath(sub_dir.dirpath, mock_directory.dirpath) + assert rel_path not in mock_directory.items + + +def test_remove_subdirectory_by_path(mock_directory, sub_dir): + """Test removing a subdirectory by path.""" + mock_directory.remove_subdirectory_by_path(sub_dir.dirpath) + + rel_path = os.path.relpath(sub_dir.dirpath, mock_directory.dirpath) + assert rel_path not in mock_directory.items + + +def test_get_subdirectory(mock_directory, sub_dir): + """Test retrieving a subdirectory by name.""" + retrieved_subdir = mock_directory.get_subdirectory("subdir") + assert retrieved_subdir is sub_dir + + +def test_files_property(mock_directory, sub_dir, mock_codebase_graph): + """Test the 'files' property returns all files recursively.""" + all_files = mock_directory.files + assert len(all_files) == 1 + + new_file = File(filepath=Path("mock_dir/example_2.py"), G=mock_codebase_graph) + sub_dir.add_file(new_file) + + all_files = mock_directory.files + assert len(all_files) == 2 + assert new_file in all_files + + gen = mock_directory.files_generator() + files_list = list(gen) + assert len(files_list) == 2 + assert new_file in files_list + + +def test_subdirectories_property(mock_directory, sub_dir): + """Test the 'subdirectories' property returns all directories recursively.""" + all_subdirs = mock_directory.subdirectories + assert len(all_subdirs) == 1 + assert sub_dir in all_subdirs + + new_sub_dir = Directory(path=sub_dir.path / "new_subdir", dirpath=sub_dir.dirpath / "new_subdir", parent=sub_dir) + sub_dir.add_subdirectory(new_sub_dir) + + all_subdirs = mock_directory.subdirectories + assert len(all_subdirs) == 2 + assert new_sub_dir in all_subdirs + + +def test_update_filepath(mock_directory, mock_codebase_graph, mock_file): + """Test updating file paths when the directory path changes.""" + mock_directory.update_filepath("/absolute/new_mock_dir") + + # Verify the files have updated file paths + mock_codebase_graph.transaction_manager.add_file_rename_transaction.assert_called_once_with(mock_file, "/absolute/new_mock_dir/example.py") + + +def test_remove(mock_directory, sub_dir, mock_codebase_graph, mock_file): + mock_directory.remove() + + mock_codebase_graph.transaction_manager.add_file_remove_transaction.assert_called_once_with(mock_file) + + +def test_rename(mock_directory, mock_codebase_graph, mock_file): + """Test renaming the directory.""" + mock_directory.rename("renamed_dir") + # This fails because it is not implemented to rename the directory itself. + # assert mock_directory.dirpath == "/absolute/renamed_dir" + mock_codebase_graph.transaction_manager.add_file_rename_transaction.assert_called_once_with(mock_file, "renamed_dir/example.py") + + +def test_iteration(mock_directory): + """Test iterating over the directory items.""" + items = list(mock_directory) # uses Directory.__iter__ + assert len(items) == 2 + assert mock_directory.items["subdir"] in items + assert mock_directory.items["example.py"] in items + + +def test_contains(mock_directory): + """Test the containment checks using the 'in' operator.""" + assert "subdir" in mock_directory + assert "example.py" in mock_directory + + +def test_len(mock_directory): + """Test the __len__ method returns the number of items.""" + assert len(mock_directory) == 2 + + +def test_get_set_delete_item(mock_directory): + """Test __getitem__, __setitem__, and __delitem__ methods.""" + mock_file = mock_directory.items["example.py"] + mock_directory["example.py"] = mock_file + assert mock_directory["example.py"] == mock_file + + with pytest.raises(KeyError, match="subdir_2"): + del mock_directory["subdir_2"] From 0ac66018477cf743c20c77ac945524a52242d533 Mon Sep 17 00:00:00 2001 From: Chris Lee Date: Thu, 6 Feb 2025 16:47:16 -0800 Subject: [PATCH 07/17] fix: move cache utils and add unit tests --- src/codegen/sdk/core/codeowner.py | 2 +- src/codegen/sdk/core/directory.py | 2 +- .../sdk/core/interfaces/files_interface.py | 2 +- .../{git => sdk/core}/utils/cache_utils.py | 0 .../sdk/core/utils/test_cache_utils.py | 21 +++++++++++++++++++ 5 files changed, 24 insertions(+), 3 deletions(-) rename src/codegen/{git => sdk/core}/utils/cache_utils.py (100%) create mode 100644 tests/unit/codegen/sdk/core/utils/test_cache_utils.py diff --git a/src/codegen/sdk/core/codeowner.py b/src/codegen/sdk/core/codeowner.py index f494654ac..0a26a82a4 100644 --- a/src/codegen/sdk/core/codeowner.py +++ b/src/codegen/sdk/core/codeowner.py @@ -4,8 +4,8 @@ from codeowners import CodeOwners as CodeOwnersParser -from codegen.git.utils.cache_utils import cached_generator from codegen.sdk.core.interfaces.files_interface import FilesInterface, FilesParam, TClass, TFile, TFunction, TGlobalVar, TImport, TImportStatement, TSymbol +from codegen.sdk.core.utils.cache_utils import cached_generator from codegen.shared.decorators.docs import apidoc logger = logging.getLogger(__name__) diff --git a/src/codegen/sdk/core/directory.py b/src/codegen/sdk/core/directory.py index c0670adcb..5e9608f20 100644 --- a/src/codegen/sdk/core/directory.py +++ b/src/codegen/sdk/core/directory.py @@ -4,8 +4,8 @@ from pathlib import Path from typing import Generic, Self -from codegen.git.utils.cache_utils import cached_generator from codegen.sdk.core.interfaces.files_interface import FilesInterface, TClass, TFile, TFunction, TGlobalVar, TImport, TImportStatement, TSymbol +from codegen.sdk.core.utils.cache_utils import cached_generator from codegen.shared.decorators.docs import apidoc logger = logging.getLogger(__name__) diff --git a/src/codegen/sdk/core/interfaces/files_interface.py b/src/codegen/sdk/core/interfaces/files_interface.py index 71e0be29e..061b04c9d 100644 --- a/src/codegen/sdk/core/interfaces/files_interface.py +++ b/src/codegen/sdk/core/interfaces/files_interface.py @@ -3,7 +3,7 @@ from itertools import chain from typing import TYPE_CHECKING, Generic, ParamSpec, TypeVar -from codegen.git.utils.cache_utils import cached_generator +from codegen.sdk.core.utils.cache_utils import cached_generator from codegen.shared.decorators.docs import py_noapidoc if TYPE_CHECKING: diff --git a/src/codegen/git/utils/cache_utils.py b/src/codegen/sdk/core/utils/cache_utils.py similarity index 100% rename from src/codegen/git/utils/cache_utils.py rename to src/codegen/sdk/core/utils/cache_utils.py diff --git a/tests/unit/codegen/sdk/core/utils/test_cache_utils.py b/tests/unit/codegen/sdk/core/utils/test_cache_utils.py new file mode 100644 index 000000000..2075465f1 --- /dev/null +++ b/tests/unit/codegen/sdk/core/utils/test_cache_utils.py @@ -0,0 +1,21 @@ +from threading import Event + +from codegen.sdk.core.utils.cache_utils import cached_generator + + +def test_cached_generator(): + event = Event() + + @cached_generator() + def cached_function(): + assert not event.is_set() + event.set() + yield from range(10) + + # First call + result = cached_function() + assert list(result) == [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + + # Second call + result = cached_function() + assert list(result) == [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] From 1e4ea6a991221263d7e9f787c62fd606706d32d0 Mon Sep 17 00:00:00 2001 From: Chris Lee Date: Fri, 7 Feb 2025 10:51:41 -0800 Subject: [PATCH 08/17] fix: rename fileinterface to hassymbols and add noapidoc --- src/codegen/sdk/core/codeowner.py | 7 ++++--- src/codegen/sdk/core/directory.py | 7 ++++--- .../interfaces/{files_interface.py => has_symbols.py} | 4 ++-- .../sdk/core/interfaces/test_files_interface.py | 10 +++++----- 4 files changed, 15 insertions(+), 13 deletions(-) rename src/codegen/sdk/core/interfaces/{files_interface.py => has_symbols.py} (94%) diff --git a/src/codegen/sdk/core/codeowner.py b/src/codegen/sdk/core/codeowner.py index 0a26a82a4..0d40f46b4 100644 --- a/src/codegen/sdk/core/codeowner.py +++ b/src/codegen/sdk/core/codeowner.py @@ -4,15 +4,15 @@ from codeowners import CodeOwners as CodeOwnersParser -from codegen.sdk.core.interfaces.files_interface import FilesInterface, FilesParam, TClass, TFile, TFunction, TGlobalVar, TImport, TImportStatement, TSymbol +from codegen.sdk.core.interfaces.has_symbols import FilesParam, HasSymbols, TClass, TFile, TFunction, TGlobalVar, TImport, TImportStatement, TSymbol from codegen.sdk.core.utils.cache_utils import cached_generator -from codegen.shared.decorators.docs import apidoc +from codegen.shared.decorators.docs import apidoc, py_noapidoc logger = logging.getLogger(__name__) @apidoc -class CodeOwner(FilesInterface[TFile, TSymbol, TImportStatement, TGlobalVar, TClass, TFunction, TImport], Generic[TFile, TSymbol, TImportStatement, TGlobalVar, TClass, TFunction, TImport]): +class CodeOwner(HasSymbols[TFile, TSymbol, TImportStatement, TGlobalVar, TClass, TFunction, TImport], Generic[TFile, TSymbol, TImportStatement, TGlobalVar, TClass, TFunction, TImport]): """CodeOwner is a class that represents a code owner in a codebase. It is used to iterate over all files that are owned by a specific owner. @@ -50,6 +50,7 @@ def from_parser(cls, parser: CodeOwnersParser, file_source: Callable[FilesParam, codeowners.append(CodeOwner(file_source, owner_label, owner_value)) return codeowners + @py_noapidoc @cached_generator(maxsize=16) def files_generator(self, *args: FilesParam.args, **kwargs: FilesParam.kwargs) -> Iterable[TFile]: for source_file in self.files_source(*args, **kwargs): diff --git a/src/codegen/sdk/core/directory.py b/src/codegen/sdk/core/directory.py index 5e9608f20..9f31bb1b8 100644 --- a/src/codegen/sdk/core/directory.py +++ b/src/codegen/sdk/core/directory.py @@ -4,15 +4,15 @@ from pathlib import Path from typing import Generic, Self -from codegen.sdk.core.interfaces.files_interface import FilesInterface, TClass, TFile, TFunction, TGlobalVar, TImport, TImportStatement, TSymbol +from codegen.sdk.core.interfaces.has_symbols import HasSymbols, TClass, TFile, TFunction, TGlobalVar, TImport, TImportStatement, TSymbol from codegen.sdk.core.utils.cache_utils import cached_generator -from codegen.shared.decorators.docs import apidoc +from codegen.shared.decorators.docs import apidoc, py_noapidoc logger = logging.getLogger(__name__) @apidoc -class Directory(FilesInterface[TFile, TSymbol, TImportStatement, TGlobalVar, TClass, TFunction, TImport], Generic[TFile, TSymbol, TImportStatement, TGlobalVar, TClass, TFunction, TImport]): +class Directory(HasSymbols[TFile, TSymbol, TImportStatement, TGlobalVar, TClass, TFunction, TImport], Generic[TFile, TSymbol, TImportStatement, TGlobalVar, TClass, TFunction, TImport]): """Directory representation for codebase. GraphSitter abstraction of a file directory that can be used to look for files and symbols within a specific directory. @@ -101,6 +101,7 @@ def _get_subdirectories(directory: Directory): _get_subdirectories(self) return subdirectories + @py_noapidoc @cached_generator() def files_generator(self) -> Iterator[TFile]: """Yield files recursively from the directory.""" diff --git a/src/codegen/sdk/core/interfaces/files_interface.py b/src/codegen/sdk/core/interfaces/has_symbols.py similarity index 94% rename from src/codegen/sdk/core/interfaces/files_interface.py rename to src/codegen/sdk/core/interfaces/has_symbols.py index 061b04c9d..4b4913416 100644 --- a/src/codegen/sdk/core/interfaces/files_interface.py +++ b/src/codegen/sdk/core/interfaces/has_symbols.py @@ -36,7 +36,7 @@ TSGlobalVar = TypeVar("TSGlobalVar", bound="Assignment") -class FilesInterface(Generic[TFile, TSymbol, TImportStatement, TGlobalVar, TClass, TFunction, TImport]): +class HasSymbols(Generic[TFile, TSymbol, TImportStatement, TGlobalVar, TClass, TFunction, TImport]): """Abstract interface for files in a codebase. Abstract interface for files in a codebase. @@ -104,7 +104,7 @@ def get_function(self, name: str) -> TFunction | None: return next((s for s in self.functions if s.name == name), None) @py_noapidoc - def get_export(self: "FilesInterface[TSFile, TSSymbol, TSImportStatement, TSGlobalVar, TSClass, TSFunction, TSImport]", name: str) -> "TSExport | None": + def get_export(self: "HasSymbols[TSFile, TSSymbol, TSImportStatement, TSGlobalVar, TSClass, TSFunction, TSImport]", name: str) -> "TSExport | None": """Get an export by name in files container (supports only typescript).""" return next((s for s in self.exports if s.name == name), None) diff --git a/tests/unit/codegen/sdk/core/interfaces/test_files_interface.py b/tests/unit/codegen/sdk/core/interfaces/test_files_interface.py index b8817af80..d2e38d94d 100644 --- a/tests/unit/codegen/sdk/core/interfaces/test_files_interface.py +++ b/tests/unit/codegen/sdk/core/interfaces/test_files_interface.py @@ -2,12 +2,12 @@ import pytest -from codegen.sdk.core.interfaces.files_interface import FilesInterface +from codegen.sdk.core.interfaces.has_symbols import HasSymbols @pytest.fixture def fake_interface(): - class FakeFilesInterface(FilesInterface): + class FakeHasSymbols(HasSymbols): def __init__(self, files): self._files = files @@ -52,12 +52,12 @@ def files_generator(self, *args, **kwargs): file2.imports[0].name = "import2" fake_files = [file1, file2] - return FakeFilesInterface(fake_files) + return FakeHasSymbols(fake_files) def test_files_generator_not_implemented(): - # Instantiating FilesInterface directly should cause files_generator to raise NotImplementedError. - fi = FilesInterface() + # Instantiating HasSymbols directly should cause files_generator to raise NotImplementedError. + fi = HasSymbols() with pytest.raises(NotImplementedError): list(fi.files_generator()) From 6375e19d1796f80e36f730d23fb07dc9756a1cce Mon Sep 17 00:00:00 2001 From: Chris Lee Date: Fri, 7 Feb 2025 12:22:53 -0800 Subject: [PATCH 09/17] add: files property test --- src/codegen/sdk/core/codeowner.py | 6 +++++- tests/unit/codegen/sdk/core/test_codeowner.py | 17 +++++++++++++++-- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/src/codegen/sdk/core/codeowner.py b/src/codegen/sdk/core/codeowner.py index 0d40f46b4..c1744578d 100644 --- a/src/codegen/sdk/core/codeowner.py +++ b/src/codegen/sdk/core/codeowner.py @@ -4,6 +4,7 @@ from codeowners import CodeOwners as CodeOwnersParser +from codegen.sdk._proxy import proxy_property from codegen.sdk.core.interfaces.has_symbols import FilesParam, HasSymbols, TClass, TFile, TFunction, TGlobalVar, TImport, TImportStatement, TSymbol from codegen.sdk.core.utils.cache_utils import cached_generator from codegen.shared.decorators.docs import apidoc, py_noapidoc @@ -31,7 +32,6 @@ def __init__(self, files_source: Callable[FilesParam, Iterable[TFile]], owner_ty self.owner_type = owner_type self.owner_value = owner_value self.files_source = files_source - self.files = self.files_generator @classmethod def from_parser(cls, parser: CodeOwnersParser, file_source: Callable[FilesParam, Iterable[TFile]]) -> list["CodeOwner"]: @@ -58,6 +58,10 @@ def files_generator(self, *args: FilesParam.args, **kwargs: FilesParam.kwargs) - if self.owner_value in source_file.owners: yield source_file + @proxy_property + def files(self, *args: FilesParam.args, **kwargs: FilesParam.kwargs) -> Iterable[TFile]: + return self.files_generator(*args, **kwargs) + @property def name(self) -> str: return self.owner_value diff --git a/tests/unit/codegen/sdk/core/test_codeowner.py b/tests/unit/codegen/sdk/core/test_codeowner.py index fc740bd47..c075c9fe0 100644 --- a/tests/unit/codegen/sdk/core/test_codeowner.py +++ b/tests/unit/codegen/sdk/core/test_codeowner.py @@ -1,4 +1,3 @@ -from collections.abc import Iterable from unittest.mock import MagicMock import pytest @@ -8,7 +7,7 @@ # Dummy file objects used for testing CodeOwner. @pytest.fixture -def fake_files() -> Iterable[MagicMock]: +def fake_files() -> list[MagicMock]: file1 = MagicMock() file1.owners = ["alice", "bob"] @@ -33,6 +32,20 @@ def file_source(*args, **kwargs): assert fake_files[1] not in files +def test_files_property(fake_files): + def file_source(*args, **kwargs): + return fake_files + + codeowner = CodeOwner(file_source, "USERNAME", "alice") + files = list(codeowner.files) + # file1 and file3 contain "alice" as one of their owners. + assert fake_files[0] in files + assert fake_files[2] in files + assert fake_files[1] not in files + + assert files == list(codeowner.files()) + + def test_name_property_and_repr(): def dummy_source(*args, **kwargs): return [] From 28ffbb507077467fa4c5d8d0a7b8ca785775f562 Mon Sep 17 00:00:00 2001 From: Chris Lee Date: Fri, 7 Feb 2025 15:50:21 -0800 Subject: [PATCH 10/17] fix: implement uncache system integrated lru_cache --- src/codegen/sdk/core/utils/cache_utils.py | 4 +- src/codegen/sdk/extensions/utils.pyi | 6 ++- src/codegen/sdk/extensions/utils.pyx | 23 ++++++++++- tests/unit/codegen/extensions/test_utils.py | 43 +++++++++++++++++++++ 4 files changed, 71 insertions(+), 5 deletions(-) create mode 100644 tests/unit/codegen/extensions/test_utils.py diff --git a/src/codegen/sdk/core/utils/cache_utils.py b/src/codegen/sdk/core/utils/cache_utils.py index 784bf0dc0..60f7c4dbf 100644 --- a/src/codegen/sdk/core/utils/cache_utils.py +++ b/src/codegen/sdk/core/utils/cache_utils.py @@ -2,6 +2,8 @@ from collections.abc import Iterator from typing import Callable, Generic, ParamSpec, TypeVar +from codegen.sdk.extensions.utils import lru_cache + ItemType = TypeVar("ItemType") GenParamSpec = ParamSpec("GenParamSpec") @@ -33,7 +35,7 @@ def cached_generator(maxsize: int = 16, typed: bool = False) -> Callable[[Callab """ def decorator(func: Callable[GenParamSpec, Iterator[ItemType]]) -> Callable[GenParamSpec, Iterator[ItemType]]: - @functools.lru_cache(maxsize=maxsize, typed=typed) + @lru_cache(maxsize=maxsize, typed=typed) @functools.wraps(func) def wrapper(*args: GenParamSpec.args, **kwargs: GenParamSpec.kwargs) -> Iterator[ItemType]: return LazyGeneratorCache(func(*args, **kwargs)) diff --git a/src/codegen/sdk/extensions/utils.pyi b/src/codegen/sdk/extensions/utils.pyi index 67e9d92ea..952bdd0ef 100644 --- a/src/codegen/sdk/extensions/utils.pyi +++ b/src/codegen/sdk/extensions/utils.pyi @@ -1,5 +1,6 @@ from collections.abc import Generator, Iterable -from functools import cached_property +from functools import cached_property as functools_cached_property +from functools import lru_cache as functools_lru_cache from tree_sitter import Node as TSNode @@ -18,7 +19,8 @@ def find_line_start_and_end_nodes(node: TSNode) -> list[tuple[TSNode, TSNode]]: def find_first_descendant(node: TSNode, type_names: list[str], max_depth: int | None = None) -> TSNode | None: ... -cached_property = cached_property +cached_property = functools_cached_property +lru_cache = functools_lru_cache def uncache_all(): ... def is_descendant_of(node: TSNode, possible_parent: TSNode) -> bool: ... diff --git a/src/codegen/sdk/extensions/utils.pyx b/src/codegen/sdk/extensions/utils.pyx index e95b69ef4..992db3663 100644 --- a/src/codegen/sdk/extensions/utils.pyx +++ b/src/codegen/sdk/extensions/utils.pyx @@ -1,6 +1,7 @@ from collections import Counter from collections.abc import Generator, Iterable -from functools import cached_property +from functools import cached_property as functools_cached_property +from functools import lru_cache as functools_lru_cache from tabulate import tabulate from tree_sitter import Node as TSNode @@ -106,10 +107,11 @@ def find_first_descendant(node: TSNode, type_names: list[str], max_depth: int | to_uncache = [] +lru_caches = [] counter = Counter() -class cached_property(cached_property): +class cached_property(functools_cached_property): def __get__(self, instance, owner=None): ret = super().__get__(instance) if instance is not None: @@ -118,6 +120,20 @@ class cached_property(cached_property): return ret +def lru_cache(func=None, *, maxsize=128, typed=False): + """A wrapper around functools.lru_cache that tracks the cached function so that its cache + can be cleared later via uncache_all(). + """ + if func is None: + # return decorator + return lambda f: lru_cache(f, maxsize=maxsize, typed=typed) + + # return decorated + cached_func = functools_lru_cache(maxsize=maxsize, typed=typed)(func) + lru_caches.append(cached_func) + return cached_func + + def uncache_all(): for instance, name in to_uncache: try: @@ -125,6 +141,9 @@ def uncache_all(): except KeyError: pass + for cached_func in lru_caches: + cached_func.cache_clear() + def report(): print(tabulate(counter.most_common(10))) diff --git a/tests/unit/codegen/extensions/test_utils.py b/tests/unit/codegen/extensions/test_utils.py new file mode 100644 index 000000000..395d1905a --- /dev/null +++ b/tests/unit/codegen/extensions/test_utils.py @@ -0,0 +1,43 @@ +from threading import Event + +import pytest + +from codegen.sdk.extensions.utils import lru_cache, uncache_all + + +def test_lru_cache_with_uncache_all(): + event = Event() + + @lru_cache + def cached_function(): + assert not event.is_set() + event.set() + return 42 + + assert cached_function() == 42 + assert cached_function() == 42 + + uncache_all() + + with pytest.raises(AssertionError): + cached_function() + + +def test_lru_cache_args_with_uncache_all(): + event = [Event() for _ in range(2)] + + @lru_cache(maxsize=2) + def cached_function(a): + assert not event[a].is_set() + event[a].set() + return a + + for _ in range(2): + for idx in range(2): + assert cached_function(idx) == idx + + uncache_all() + + for idx in range(2): + with pytest.raises(AssertionError): + cached_function(idx) From 30f0cce0f84e6479b9530bc330a13ec989fff75d Mon Sep 17 00:00:00 2001 From: Chris Lee Date: Mon, 10 Feb 2025 10:41:43 -0800 Subject: [PATCH 11/17] fix: move py_noapidoc in dec stack --- src/codegen/sdk/core/codeowner.py | 46 +++++++++++++++---- .../sdk/core/interfaces/has_symbols.py | 18 ++++++-- 2 files changed, 52 insertions(+), 12 deletions(-) diff --git a/src/codegen/sdk/core/codeowner.py b/src/codegen/sdk/core/codeowner.py index c1744578d..b89b85b25 100644 --- a/src/codegen/sdk/core/codeowner.py +++ b/src/codegen/sdk/core/codeowner.py @@ -5,7 +5,17 @@ from codeowners import CodeOwners as CodeOwnersParser from codegen.sdk._proxy import proxy_property -from codegen.sdk.core.interfaces.has_symbols import FilesParam, HasSymbols, TClass, TFile, TFunction, TGlobalVar, TImport, TImportStatement, TSymbol +from codegen.sdk.core.interfaces.has_symbols import ( + FilesParam, + HasSymbols, + TClass, + TFile, + TFunction, + TGlobalVar, + TImport, + TImportStatement, + TSymbol, +) from codegen.sdk.core.utils.cache_utils import cached_generator from codegen.shared.decorators.docs import apidoc, py_noapidoc @@ -13,7 +23,12 @@ @apidoc -class CodeOwner(HasSymbols[TFile, TSymbol, TImportStatement, TGlobalVar, TClass, TFunction, TImport], Generic[TFile, TSymbol, TImportStatement, TGlobalVar, TClass, TFunction, TImport]): +class CodeOwner( + HasSymbols[ + TFile, TSymbol, TImportStatement, TGlobalVar, TClass, TFunction, TImport + ], + Generic[TFile, TSymbol, TImportStatement, TGlobalVar, TClass, TFunction, TImport], +): """CodeOwner is a class that represents a code owner in a codebase. It is used to iterate over all files that are owned by a specific owner. @@ -28,13 +43,22 @@ class CodeOwner(HasSymbols[TFile, TSymbol, TImportStatement, TGlobalVar, TClass, owner_value: str files_source: Callable[FilesParam, Iterable[TFile]] - def __init__(self, files_source: Callable[FilesParam, Iterable[TFile]], owner_type: Literal["USERNAME", "TEAM", "EMAIL"], owner_value: str): + def __init__( + self, + files_source: Callable[FilesParam, Iterable[TFile]], + owner_type: Literal["USERNAME", "TEAM", "EMAIL"], + owner_value: str, + ): self.owner_type = owner_type self.owner_value = owner_value self.files_source = files_source @classmethod - def from_parser(cls, parser: CodeOwnersParser, file_source: Callable[FilesParam, Iterable[TFile]]) -> list["CodeOwner"]: + def from_parser( + cls, + parser: CodeOwnersParser, + file_source: Callable[FilesParam, Iterable[TFile]], + ) -> list["CodeOwner"]: """Create a list of CodeOwner objects from a CodeOwnersParser. Args: @@ -50,16 +74,20 @@ def from_parser(cls, parser: CodeOwnersParser, file_source: Callable[FilesParam, codeowners.append(CodeOwner(file_source, owner_label, owner_value)) return codeowners - @py_noapidoc @cached_generator(maxsize=16) - def files_generator(self, *args: FilesParam.args, **kwargs: FilesParam.kwargs) -> Iterable[TFile]: + @py_noapidoc + def files_generator( + self, *args: FilesParam.args, **kwargs: FilesParam.kwargs + ) -> Iterable[TFile]: for source_file in self.files_source(*args, **kwargs): # Filter files by owner value if self.owner_value in source_file.owners: yield source_file @proxy_property - def files(self, *args: FilesParam.args, **kwargs: FilesParam.kwargs) -> Iterable[TFile]: + def files( + self, *args: FilesParam.args, **kwargs: FilesParam.kwargs + ) -> Iterable[TFile]: return self.files_generator(*args, **kwargs) @property @@ -70,4 +98,6 @@ def __iter__(self) -> Iterator[TFile]: return iter(self.files_generator()) def __repr__(self) -> str: - return f"CodeOwner(owner_type={self.owner_type}, owner_value={self.owner_value})" + return ( + f"CodeOwner(owner_type={self.owner_type}, owner_value={self.owner_value})" + ) diff --git a/src/codegen/sdk/core/interfaces/has_symbols.py b/src/codegen/sdk/core/interfaces/has_symbols.py index 4b4913416..fe3fc2448 100644 --- a/src/codegen/sdk/core/interfaces/has_symbols.py +++ b/src/codegen/sdk/core/interfaces/has_symbols.py @@ -36,14 +36,19 @@ TSGlobalVar = TypeVar("TSGlobalVar", bound="Assignment") -class HasSymbols(Generic[TFile, TSymbol, TImportStatement, TGlobalVar, TClass, TFunction, TImport]): +class HasSymbols( + Generic[TFile, TSymbol, TImportStatement, TGlobalVar, TClass, TFunction, TImport] +): """Abstract interface for files in a codebase. Abstract interface for files in a codebase. """ @cached_generator() - def files_generator(self, *args: FilesParam.args, **kwargs: FilesParam.kwargs) -> Iterator[TFile]: + def files_generator( + self, *args: FilesParam.args, **kwargs: FilesParam.kwargs + ) -> Iterator[TFile]: + """Generator for yielding files of the current container's scope.""" msg = "This method should be implemented by the subclass" raise NotImplementedError(msg) @@ -55,7 +60,9 @@ def symbols(self) -> list[TSymbol]: @property def import_statements(self) -> list[TImportStatement]: """Get a recursive list of all import statements in files container.""" - return list(chain.from_iterable(f.import_statements for f in self.files_generator())) + return list( + chain.from_iterable(f.import_statements for f in self.files_generator()) + ) @property def global_vars(self) -> list[TGlobalVar]: @@ -104,7 +111,10 @@ def get_function(self, name: str) -> TFunction | None: return next((s for s in self.functions if s.name == name), None) @py_noapidoc - def get_export(self: "HasSymbols[TSFile, TSSymbol, TSImportStatement, TSGlobalVar, TSClass, TSFunction, TSImport]", name: str) -> "TSExport | None": + def get_export( + self: "HasSymbols[TSFile, TSSymbol, TSImportStatement, TSGlobalVar, TSClass, TSFunction, TSImport]", + name: str, + ) -> "TSExport | None": """Get an export by name in files container (supports only typescript).""" return next((s for s in self.exports if s.name == name), None) From 6e7d1974d4d0095d02286c88418ea729d7d80561 Mon Sep 17 00:00:00 2001 From: clee-codegen <185840274+clee-codegen@users.noreply.github.com> Date: Mon, 10 Feb 2025 18:42:40 +0000 Subject: [PATCH 12/17] Automated pre-commit update --- src/codegen/sdk/core/codeowner.py | 16 ++++------------ src/codegen/sdk/core/interfaces/has_symbols.py | 12 +++--------- 2 files changed, 7 insertions(+), 21 deletions(-) diff --git a/src/codegen/sdk/core/codeowner.py b/src/codegen/sdk/core/codeowner.py index b89b85b25..e1cf24d21 100644 --- a/src/codegen/sdk/core/codeowner.py +++ b/src/codegen/sdk/core/codeowner.py @@ -24,9 +24,7 @@ @apidoc class CodeOwner( - HasSymbols[ - TFile, TSymbol, TImportStatement, TGlobalVar, TClass, TFunction, TImport - ], + HasSymbols[TFile, TSymbol, TImportStatement, TGlobalVar, TClass, TFunction, TImport], Generic[TFile, TSymbol, TImportStatement, TGlobalVar, TClass, TFunction, TImport], ): """CodeOwner is a class that represents a code owner in a codebase. @@ -76,18 +74,14 @@ def from_parser( @cached_generator(maxsize=16) @py_noapidoc - def files_generator( - self, *args: FilesParam.args, **kwargs: FilesParam.kwargs - ) -> Iterable[TFile]: + def files_generator(self, *args: FilesParam.args, **kwargs: FilesParam.kwargs) -> Iterable[TFile]: for source_file in self.files_source(*args, **kwargs): # Filter files by owner value if self.owner_value in source_file.owners: yield source_file @proxy_property - def files( - self, *args: FilesParam.args, **kwargs: FilesParam.kwargs - ) -> Iterable[TFile]: + def files(self, *args: FilesParam.args, **kwargs: FilesParam.kwargs) -> Iterable[TFile]: return self.files_generator(*args, **kwargs) @property @@ -98,6 +92,4 @@ def __iter__(self) -> Iterator[TFile]: return iter(self.files_generator()) def __repr__(self) -> str: - return ( - f"CodeOwner(owner_type={self.owner_type}, owner_value={self.owner_value})" - ) + return f"CodeOwner(owner_type={self.owner_type}, owner_value={self.owner_value})" diff --git a/src/codegen/sdk/core/interfaces/has_symbols.py b/src/codegen/sdk/core/interfaces/has_symbols.py index fe3fc2448..7daefa99b 100644 --- a/src/codegen/sdk/core/interfaces/has_symbols.py +++ b/src/codegen/sdk/core/interfaces/has_symbols.py @@ -36,18 +36,14 @@ TSGlobalVar = TypeVar("TSGlobalVar", bound="Assignment") -class HasSymbols( - Generic[TFile, TSymbol, TImportStatement, TGlobalVar, TClass, TFunction, TImport] -): +class HasSymbols(Generic[TFile, TSymbol, TImportStatement, TGlobalVar, TClass, TFunction, TImport]): """Abstract interface for files in a codebase. Abstract interface for files in a codebase. """ @cached_generator() - def files_generator( - self, *args: FilesParam.args, **kwargs: FilesParam.kwargs - ) -> Iterator[TFile]: + def files_generator(self, *args: FilesParam.args, **kwargs: FilesParam.kwargs) -> Iterator[TFile]: """Generator for yielding files of the current container's scope.""" msg = "This method should be implemented by the subclass" raise NotImplementedError(msg) @@ -60,9 +56,7 @@ def symbols(self) -> list[TSymbol]: @property def import_statements(self) -> list[TImportStatement]: """Get a recursive list of all import statements in files container.""" - return list( - chain.from_iterable(f.import_statements for f in self.files_generator()) - ) + return list(chain.from_iterable(f.import_statements for f in self.files_generator())) @property def global_vars(self) -> list[TGlobalVar]: From fe659e87cd2f38fbd7dac2de9ae85ffd95344ff5 Mon Sep 17 00:00:00 2001 From: Chris Lee Date: Mon, 10 Feb 2025 10:59:23 -0800 Subject: [PATCH 13/17] fix: use noapidoc instead of py_noapidoc --- src/codegen/sdk/core/codeowner.py | 20 ++++++++++++------ src/codegen/sdk/core/directory.py | 35 +++++++++++++++++++++++++------ 2 files changed, 43 insertions(+), 12 deletions(-) diff --git a/src/codegen/sdk/core/codeowner.py b/src/codegen/sdk/core/codeowner.py index e1cf24d21..2d1814c3e 100644 --- a/src/codegen/sdk/core/codeowner.py +++ b/src/codegen/sdk/core/codeowner.py @@ -17,14 +17,16 @@ TSymbol, ) from codegen.sdk.core.utils.cache_utils import cached_generator -from codegen.shared.decorators.docs import apidoc, py_noapidoc +from codegen.shared.decorators.docs import apidoc, noapidoc logger = logging.getLogger(__name__) @apidoc class CodeOwner( - HasSymbols[TFile, TSymbol, TImportStatement, TGlobalVar, TClass, TFunction, TImport], + HasSymbols[ + TFile, TSymbol, TImportStatement, TGlobalVar, TClass, TFunction, TImport + ], Generic[TFile, TSymbol, TImportStatement, TGlobalVar, TClass, TFunction, TImport], ): """CodeOwner is a class that represents a code owner in a codebase. @@ -73,15 +75,19 @@ def from_parser( return codeowners @cached_generator(maxsize=16) - @py_noapidoc - def files_generator(self, *args: FilesParam.args, **kwargs: FilesParam.kwargs) -> Iterable[TFile]: + @noapidoc + def files_generator( + self, *args: FilesParam.args, **kwargs: FilesParam.kwargs + ) -> Iterable[TFile]: for source_file in self.files_source(*args, **kwargs): # Filter files by owner value if self.owner_value in source_file.owners: yield source_file @proxy_property - def files(self, *args: FilesParam.args, **kwargs: FilesParam.kwargs) -> Iterable[TFile]: + def files( + self, *args: FilesParam.args, **kwargs: FilesParam.kwargs + ) -> Iterable[TFile]: return self.files_generator(*args, **kwargs) @property @@ -92,4 +98,6 @@ def __iter__(self) -> Iterator[TFile]: return iter(self.files_generator()) def __repr__(self) -> str: - return f"CodeOwner(owner_type={self.owner_type}, owner_value={self.owner_value})" + return ( + f"CodeOwner(owner_type={self.owner_type}, owner_value={self.owner_value})" + ) diff --git a/src/codegen/sdk/core/directory.py b/src/codegen/sdk/core/directory.py index 9f31bb1b8..4715ed74c 100644 --- a/src/codegen/sdk/core/directory.py +++ b/src/codegen/sdk/core/directory.py @@ -4,15 +4,29 @@ from pathlib import Path from typing import Generic, Self -from codegen.sdk.core.interfaces.has_symbols import HasSymbols, TClass, TFile, TFunction, TGlobalVar, TImport, TImportStatement, TSymbol +from codegen.sdk.core.interfaces.has_symbols import ( + HasSymbols, + TClass, + TFile, + TFunction, + TGlobalVar, + TImport, + TImportStatement, + TSymbol, +) from codegen.sdk.core.utils.cache_utils import cached_generator -from codegen.shared.decorators.docs import apidoc, py_noapidoc +from codegen.shared.decorators.docs import apidoc, noapidoc logger = logging.getLogger(__name__) @apidoc -class Directory(HasSymbols[TFile, TSymbol, TImportStatement, TGlobalVar, TClass, TFunction, TImport], Generic[TFile, TSymbol, TImportStatement, TGlobalVar, TClass, TFunction, TImport]): +class Directory( + HasSymbols[ + TFile, TSymbol, TImportStatement, TGlobalVar, TClass, TFunction, TImport + ], + Generic[TFile, TSymbol, TImportStatement, TGlobalVar, TClass, TFunction, TImport], +): """Directory representation for codebase. GraphSitter abstraction of a file directory that can be used to look for files and symbols within a specific directory. @@ -101,7 +115,7 @@ def _get_subdirectories(directory: Directory): _get_subdirectories(self) return subdirectories - @py_noapidoc + @noapidoc @cached_generator() def files_generator(self) -> Iterator[TFile]: """Yield files recursively from the directory.""" @@ -128,7 +142,14 @@ def get_file(self, filename: str, ignore_case: bool = False) -> TFile | None: from codegen.sdk.core.file import File if ignore_case: - return next((f for name, f in self.items.items() if name.lower() == filename.lower() and isinstance(f, File)), None) + return next( + ( + f + for name, f in self.items.items() + if name.lower() == filename.lower() and isinstance(f, File) + ), + None, + ) return self.items.get(filename, None) def add_subdirectory(self, subdirectory: Self) -> None: @@ -155,7 +176,9 @@ def update_filepath(self, new_filepath: str) -> None: old_path = self.dirpath new_path = new_filepath for file in self.files: - new_file_path = os.path.join(new_path, os.path.relpath(file.file_path, old_path)) + new_file_path = os.path.join( + new_path, os.path.relpath(file.file_path, old_path) + ) file.update_filepath(new_file_path) def remove(self) -> None: From ceaba3fdd81b4968d647b74fd27582c959636a02 Mon Sep 17 00:00:00 2001 From: clee-codegen <185840274+clee-codegen@users.noreply.github.com> Date: Mon, 10 Feb 2025 19:00:47 +0000 Subject: [PATCH 14/17] Automated pre-commit update --- src/codegen/sdk/core/codeowner.py | 16 ++++------------ src/codegen/sdk/core/directory.py | 14 +++----------- 2 files changed, 7 insertions(+), 23 deletions(-) diff --git a/src/codegen/sdk/core/codeowner.py b/src/codegen/sdk/core/codeowner.py index 2d1814c3e..c389e7a3b 100644 --- a/src/codegen/sdk/core/codeowner.py +++ b/src/codegen/sdk/core/codeowner.py @@ -24,9 +24,7 @@ @apidoc class CodeOwner( - HasSymbols[ - TFile, TSymbol, TImportStatement, TGlobalVar, TClass, TFunction, TImport - ], + HasSymbols[TFile, TSymbol, TImportStatement, TGlobalVar, TClass, TFunction, TImport], Generic[TFile, TSymbol, TImportStatement, TGlobalVar, TClass, TFunction, TImport], ): """CodeOwner is a class that represents a code owner in a codebase. @@ -76,18 +74,14 @@ def from_parser( @cached_generator(maxsize=16) @noapidoc - def files_generator( - self, *args: FilesParam.args, **kwargs: FilesParam.kwargs - ) -> Iterable[TFile]: + def files_generator(self, *args: FilesParam.args, **kwargs: FilesParam.kwargs) -> Iterable[TFile]: for source_file in self.files_source(*args, **kwargs): # Filter files by owner value if self.owner_value in source_file.owners: yield source_file @proxy_property - def files( - self, *args: FilesParam.args, **kwargs: FilesParam.kwargs - ) -> Iterable[TFile]: + def files(self, *args: FilesParam.args, **kwargs: FilesParam.kwargs) -> Iterable[TFile]: return self.files_generator(*args, **kwargs) @property @@ -98,6 +92,4 @@ def __iter__(self) -> Iterator[TFile]: return iter(self.files_generator()) def __repr__(self) -> str: - return ( - f"CodeOwner(owner_type={self.owner_type}, owner_value={self.owner_value})" - ) + return f"CodeOwner(owner_type={self.owner_type}, owner_value={self.owner_value})" diff --git a/src/codegen/sdk/core/directory.py b/src/codegen/sdk/core/directory.py index 4715ed74c..115a28062 100644 --- a/src/codegen/sdk/core/directory.py +++ b/src/codegen/sdk/core/directory.py @@ -22,9 +22,7 @@ @apidoc class Directory( - HasSymbols[ - TFile, TSymbol, TImportStatement, TGlobalVar, TClass, TFunction, TImport - ], + HasSymbols[TFile, TSymbol, TImportStatement, TGlobalVar, TClass, TFunction, TImport], Generic[TFile, TSymbol, TImportStatement, TGlobalVar, TClass, TFunction, TImport], ): """Directory representation for codebase. @@ -143,11 +141,7 @@ def get_file(self, filename: str, ignore_case: bool = False) -> TFile | None: if ignore_case: return next( - ( - f - for name, f in self.items.items() - if name.lower() == filename.lower() and isinstance(f, File) - ), + (f for name, f in self.items.items() if name.lower() == filename.lower() and isinstance(f, File)), None, ) return self.items.get(filename, None) @@ -176,9 +170,7 @@ def update_filepath(self, new_filepath: str) -> None: old_path = self.dirpath new_path = new_filepath for file in self.files: - new_file_path = os.path.join( - new_path, os.path.relpath(file.file_path, old_path) - ) + new_file_path = os.path.join(new_path, os.path.relpath(file.file_path, old_path)) file.update_filepath(new_file_path) def remove(self) -> None: From fa302eb200bbde19ec9d4be8e03036f722042ff6 Mon Sep 17 00:00:00 2001 From: Chris Lee Date: Mon, 10 Feb 2025 13:57:56 -0800 Subject: [PATCH 15/17] add: docstring codeowner.files --- src/codegen/sdk/core/codeowner.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/src/codegen/sdk/core/codeowner.py b/src/codegen/sdk/core/codeowner.py index c389e7a3b..7500d38c5 100644 --- a/src/codegen/sdk/core/codeowner.py +++ b/src/codegen/sdk/core/codeowner.py @@ -24,7 +24,9 @@ @apidoc class CodeOwner( - HasSymbols[TFile, TSymbol, TImportStatement, TGlobalVar, TClass, TFunction, TImport], + HasSymbols[ + TFile, TSymbol, TImportStatement, TGlobalVar, TClass, TFunction, TImport + ], Generic[TFile, TSymbol, TImportStatement, TGlobalVar, TClass, TFunction, TImport], ): """CodeOwner is a class that represents a code owner in a codebase. @@ -74,14 +76,19 @@ def from_parser( @cached_generator(maxsize=16) @noapidoc - def files_generator(self, *args: FilesParam.args, **kwargs: FilesParam.kwargs) -> Iterable[TFile]: + def files_generator( + self, *args: FilesParam.args, **kwargs: FilesParam.kwargs + ) -> Iterable[TFile]: for source_file in self.files_source(*args, **kwargs): # Filter files by owner value if self.owner_value in source_file.owners: yield source_file @proxy_property - def files(self, *args: FilesParam.args, **kwargs: FilesParam.kwargs) -> Iterable[TFile]: + def files( + self, *args: FilesParam.args, **kwargs: FilesParam.kwargs + ) -> Iterable[TFile]: + """Recursively iterate over all files in the codebase that are owned by the current code owner.""" return self.files_generator(*args, **kwargs) @property @@ -92,4 +99,6 @@ def __iter__(self) -> Iterator[TFile]: return iter(self.files_generator()) def __repr__(self) -> str: - return f"CodeOwner(owner_type={self.owner_type}, owner_value={self.owner_value})" + return ( + f"CodeOwner(owner_type={self.owner_type}, owner_value={self.owner_value})" + ) From 34756adc4105ceddb66288f393d4ccbd6b6461b6 Mon Sep 17 00:00:00 2001 From: clee-codegen <185840274+clee-codegen@users.noreply.github.com> Date: Mon, 10 Feb 2025 21:58:47 +0000 Subject: [PATCH 16/17] Automated pre-commit update --- src/codegen/sdk/core/codeowner.py | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/src/codegen/sdk/core/codeowner.py b/src/codegen/sdk/core/codeowner.py index 7500d38c5..973e335b6 100644 --- a/src/codegen/sdk/core/codeowner.py +++ b/src/codegen/sdk/core/codeowner.py @@ -24,9 +24,7 @@ @apidoc class CodeOwner( - HasSymbols[ - TFile, TSymbol, TImportStatement, TGlobalVar, TClass, TFunction, TImport - ], + HasSymbols[TFile, TSymbol, TImportStatement, TGlobalVar, TClass, TFunction, TImport], Generic[TFile, TSymbol, TImportStatement, TGlobalVar, TClass, TFunction, TImport], ): """CodeOwner is a class that represents a code owner in a codebase. @@ -76,18 +74,14 @@ def from_parser( @cached_generator(maxsize=16) @noapidoc - def files_generator( - self, *args: FilesParam.args, **kwargs: FilesParam.kwargs - ) -> Iterable[TFile]: + def files_generator(self, *args: FilesParam.args, **kwargs: FilesParam.kwargs) -> Iterable[TFile]: for source_file in self.files_source(*args, **kwargs): # Filter files by owner value if self.owner_value in source_file.owners: yield source_file @proxy_property - def files( - self, *args: FilesParam.args, **kwargs: FilesParam.kwargs - ) -> Iterable[TFile]: + def files(self, *args: FilesParam.args, **kwargs: FilesParam.kwargs) -> Iterable[TFile]: """Recursively iterate over all files in the codebase that are owned by the current code owner.""" return self.files_generator(*args, **kwargs) @@ -99,6 +93,4 @@ def __iter__(self) -> Iterator[TFile]: return iter(self.files_generator()) def __repr__(self) -> str: - return ( - f"CodeOwner(owner_type={self.owner_type}, owner_value={self.owner_value})" - ) + return f"CodeOwner(owner_type={self.owner_type}, owner_value={self.owner_value})" From d55d9e6b574532b8e1d95f0e077882bec1081b87 Mon Sep 17 00:00:00 2001 From: Chris Lee Date: Mon, 10 Feb 2025 21:36:21 -0800 Subject: [PATCH 17/17] add: codeowner.name docstring --- src/codegen/sdk/core/codeowner.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/codegen/sdk/core/codeowner.py b/src/codegen/sdk/core/codeowner.py index 973e335b6..bb896d83b 100644 --- a/src/codegen/sdk/core/codeowner.py +++ b/src/codegen/sdk/core/codeowner.py @@ -87,6 +87,7 @@ def files(self, *args: FilesParam.args, **kwargs: FilesParam.kwargs) -> Iterable @property def name(self) -> str: + """The name of the code owner.""" return self.owner_value def __iter__(self) -> Iterator[TFile]: