From 79a8e70b7f1f5701ff4337948c0846e15c6eb168 Mon Sep 17 00:00:00 2001 From: Carol Jung Date: Fri, 7 Feb 2025 17:59:11 -0800 Subject: [PATCH 1/5] CG-10470: Consolidate configs --- .codegen/config.toml | 25 +++++++++ pyproject.toml | 1 + src/codegen/cli/cli.py | 2 + src/codegen/cli/commands/config/main.py | 66 +++++++++++++++++++++++ src/codegen/git/configs/constants.py | 2 - src/codegen/shared/configs/config.py | 53 +++++++++++++++++++ src/codegen/shared/configs/constants.py | 11 ++++ src/codegen/shared/configs/models.py | 69 +++++++++++++++++++++++++ uv.lock | 15 ++++++ 9 files changed, 242 insertions(+), 2 deletions(-) create mode 100644 src/codegen/cli/commands/config/main.py create mode 100644 src/codegen/shared/configs/config.py create mode 100644 src/codegen/shared/configs/constants.py create mode 100644 src/codegen/shared/configs/models.py diff --git a/.codegen/config.toml b/.codegen/config.toml index 1b9d56783..f5c43ba32 100644 --- a/.codegen/config.toml +++ b/.codegen/config.toml @@ -1,2 +1,27 @@ +[secrets] +github_token = "" +openai_api_key = "" + +[repository] organization_name = "codegen-sh" repo_name = "codegen-sdk" + +[feature_flags] +syntax_highlight_enabled = false + +[feature_flags.codebase] +debug = false +verify_graph = false +track_graph = true +method_usages = true +sync_enabled = true +full_range_index = false +ignore_process_errors = true +disable_graph = false +generics = true +import_resolution_overrides = {} + +[feature_flags.codebase.typescript] +ts_dependency_manager = false +ts_language_engine = false +v8_ts_engine = false diff --git a/pyproject.toml b/pyproject.toml index 59a720f6a..603971456 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,6 +26,7 @@ dependencies = [ "watchfiles<1.1.0,>=1.0.0", "rich<14.0.0,>=13.7.1", "pydantic<3.0.0,>=2.9.2", + "pydantic-settings>=2.0.0", "docstring-parser<1.0,>=0.16", "plotly<6.0.0,>=5.24.0", "humanize<5.0.0,>=4.10.0", diff --git a/src/codegen/cli/cli.py b/src/codegen/cli/cli.py index 4cd767bd8..358a6dd38 100644 --- a/src/codegen/cli/cli.py +++ b/src/codegen/cli/cli.py @@ -1,6 +1,7 @@ import rich_click as click from rich.traceback import install +from codegen.cli.commands.config.main import config_command from codegen.cli.commands.create.main import create_command from codegen.cli.commands.deploy.main import deploy_command from codegen.cli.commands.expert.main import expert_command @@ -39,6 +40,7 @@ def main(): main.add_command(run_on_pr_command) main.add_command(notebook_command) main.add_command(reset_command) +main.add_command(config_command) if __name__ == "__main__": diff --git a/src/codegen/cli/commands/config/main.py b/src/codegen/cli/commands/config/main.py new file mode 100644 index 000000000..9ba19d6ec --- /dev/null +++ b/src/codegen/cli/commands/config/main.py @@ -0,0 +1,66 @@ +from itertools import groupby + +import rich +import rich_click as click +from rich.table import Table + + +@click.group(name="config") +def config_command(): + """Manage codegen configuration.""" + pass + + +@config_command.command(name="list") +def list_command(): + """List current configuration values.""" + from codegen.shared.configs.config import config + + table = Table(title="Configuration Values", border_style="blue", show_header=True) + table.add_column("Key", style="cyan") + table.add_column("Value", style="magenta") + + def flatten_dict(data: dict, prefix: str = "") -> dict: + items = {} + for key, value in data.items(): + full_key = f"{prefix}{key}" if prefix else key + if isinstance(value, dict): + items.update(flatten_dict(value, f"{full_key}.")) + else: + items[full_key] = value + return items + + # Get flattened config and sort by keys + flat_config = flatten_dict(config.model_dump()) + sorted_items = sorted(flat_config.items(), key=lambda x: x[0]) + + # Group by top-level prefix + def get_prefix(item): + return item[0].split(".")[0] + + for prefix, group in groupby(sorted_items, key=get_prefix): + table.add_section() + table.add_row(f"[bold blue]{prefix}[/bold blue]", "") + for key, value in group: + # Remove the prefix from the displayed key + display_key = key[len(prefix) + 1 :] if "." in key else key + table.add_row(f" {display_key}", str(value)) + + rich.print(table) + + +@config_command.command(name="get") +@click.argument("key") +def get_command(key: str): + """Get a configuration value.""" + # TODO: Implement configuration get logic + rich.print(f"[yellow]Getting configuration value for: {key}[/yellow]") + + +@config_command.command(name="set") +@click.argument("key") +@click.argument("value") +def set_command(key: str, value: str): + """Set a configuration value.""" + # TODO: Implement configuration set logic + rich.print(f"[green]Setting {key}={value}[/green]") diff --git a/src/codegen/git/configs/constants.py b/src/codegen/git/configs/constants.py index 3df55ca70..d8483c60a 100644 --- a/src/codegen/git/configs/constants.py +++ b/src/codegen/git/configs/constants.py @@ -3,5 +3,3 @@ CODEGEN_BOT_NAME = "codegen-bot" CODEGEN_BOT_EMAIL = "team+codegenbot@codegen.sh" CODEOWNERS_FILEPATHS = [".github/CODEOWNERS", "CODEOWNERS", "docs/CODEOWNERS"] -HIGHSIDE_REMOTE_NAME = "highside" -LOWSIDE_REMOTE_NAME = "lowside" diff --git a/src/codegen/shared/configs/config.py b/src/codegen/shared/configs/config.py new file mode 100644 index 000000000..6e433faa1 --- /dev/null +++ b/src/codegen/shared/configs/config.py @@ -0,0 +1,53 @@ +from pathlib import Path + +import tomllib + +from codegen.shared.configs.constants import CONFIG_PATH +from codegen.shared.configs.models import Config + + +def load(config_path: Path | None = None) -> Config: + """Load configuration from the config file.""" + # Load from .env file + env_config = _load_from_env() + + # Load from .codegen/config.toml file + toml_config = _load_from_toml(config_path or CONFIG_PATH) + + # Merge configurations recursively + config_dict = _merge_configs(env_config.model_dump(), toml_config.model_dump()) + + return Config(**config_dict) + + +def _load_from_env() -> Config: + """Load configuration from the environment variables.""" + return Config() + + +def _load_from_toml(config_path: Path) -> Config: + """Load configuration from the TOML file.""" + if config_path.exists(): + with open(config_path, "rb") as f: + toml_config = tomllib.load(f) + return Config.model_validate(toml_config) + return Config() + + +def _merge_configs(base: dict, override: dict) -> dict: + """Recursively merge two dictionaries, with override taking precedence for non-null values.""" + merged = base.copy() + for key, override_value in override.items(): + if isinstance(override_value, dict) and key in base and isinstance(base[key], dict): + # Recursively merge nested dictionaries + merged[key] = _merge_configs(base[key], override_value) + elif override_value is not None and override_value != "": + # Override only if value is non-null and non-empty + merged[key] = override_value + return merged + + +config = load() + +if __name__ == "__main__": + print(config) diff --git a/src/codegen/shared/configs/constants.py b/src/codegen/shared/configs/constants.py new file mode 100644 index 000000000..d9f5d6915 --- /dev/null +++ b/src/codegen/shared/configs/constants.py @@ -0,0 +1,11 @@ +from pathlib import Path + +# Config file +CODEGEN_REPO_ROOT = Path(__file__).parent.parent.parent.parent.parent +CODEGEN_DIR_NAME = ".codegen" +CONFIG_FILENAME = "config.toml" +CONFIG_PATH = CODEGEN_REPO_ROOT / CODEGEN_DIR_NAME / CONFIG_FILENAME + +# Environment variables +ENV_FILENAME = ".env" +ENV_PATH = CODEGEN_REPO_ROOT / "src" / "codegen" / ENV_FILENAME diff --git a/src/codegen/shared/configs/models.py b/src/codegen/shared/configs/models.py new file mode 100644 index 000000000..c8f89fbc2 --- /dev/null +++ b/src/codegen/shared/configs/models.py @@ -0,0 +1,69 @@ +from pathlib import Path + +import toml +from pydantic import BaseModel, Field +from pydantic_settings import BaseSettings, SettingsConfigDict + +from codegen.shared.configs.constants import CONFIG_PATH, ENV_PATH + + +class TypescriptConfig(BaseModel): + ts_dependency_manager: bool | None = None + ts_language_engine: bool | None = None + v8_ts_engine: bool | None = None + + +class CodebaseFeatureFlags(BaseModel): + debug: bool | None = None + verify_graph: bool | None = None + track_graph: bool | None = None + method_usages: bool | None = None + sync_enabled: bool | None = None + full_range_index: bool | None = None + ignore_process_errors: bool | None = None + disable_graph: bool | None = None + generics: bool | None = None + import_resolution_overrides: dict[str, str] = Field(default_factory=lambda: {}) + typescript: TypescriptConfig = Field(default_factory=TypescriptConfig) + + +class RepositoryConfig(BaseModel): + organization_name: str | None = None + repo_name: str | None = None + + +class SecretsConfig(BaseSettings): + model_config = SettingsConfigDict( + env_prefix="CODEGEN_SECRETS__", + env_file=ENV_PATH, + case_sensitive=False, + ) + github_token: str | None = None + openai_api_key: str | None = None + + +class FeatureFlagsConfig(BaseModel): + syntax_highlight_enabled: bool | None = None + codebase: CodebaseFeatureFlags = Field(default_factory=CodebaseFeatureFlags) + + +class Config(BaseSettings): + secrets: SecretsConfig = Field(default_factory=SecretsConfig) + repository: RepositoryConfig = Field(default_factory=RepositoryConfig) + feature_flags: FeatureFlagsConfig = Field(default_factory=FeatureFlagsConfig) + + @staticmethod + def save(self, config_path: Path | None = None) -> None: + """Save configuration to the config file.""" + path = config_path or CONFIG_PATH + + path.parent.mkdir(parents=True, exist_ok=True) + + with open(path, "w") as f: + toml.dump(Config.model_dump(exclude_none=True), f) + + def __str__(self) -> str: + """Return a pretty-printed string representation of the config.""" + import json + + return json.dumps(self.model_dump(exclude_none=False), indent=2) diff --git a/uv.lock b/uv.lock index b8990d537..75eb8a701 100644 --- a/uv.lock +++ b/uv.lock @@ -377,6 +377,7 @@ dependencies = [ { name = "psutil" }, { name = "pydantic" }, { name = "pydantic-core" }, + { name = "pydantic-settings" }, { name = "pygit2" }, { name = "pygithub" }, { name = "pyinstrument" }, @@ -480,6 +481,7 @@ requires-dist = [ { name = "psutil", specifier = ">=5.8.0" }, { name = "pydantic", specifier = ">=2.9.2,<3.0.0" }, { name = "pydantic-core", specifier = ">=2.23.4" }, + { name = "pydantic-settings", specifier = ">=2.0.0" }, { name = "pygit2", specifier = ">=1.16.0" }, { name = "pygithub", specifier = "==2.5.0" }, { name = "pyinstrument", specifier = ">=5.0.0" }, @@ -1706,6 +1708,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/51/b2/b2b50d5ecf21acf870190ae5d093602d95f66c9c31f9d5de6062eb329ad1/pydantic_core-2.27.2-cp313-cp313-win_arm64.whl", hash = "sha256:ac4dbfd1691affb8f48c2c13241a2e3b60ff23247cbcf981759c768b6633cf8b", size = 1885186 }, ] +[[package]] +name = "pydantic-settings" +version = "2.7.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pydantic" }, + { name = "python-dotenv" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/73/7b/c58a586cd7d9ac66d2ee4ba60ca2d241fa837c02bca9bea80a9a8c3d22a9/pydantic_settings-2.7.1.tar.gz", hash = "sha256:10c9caad35e64bfb3c2fbf70a078c0e25cc92499782e5200747f942a065dec93", size = 79920 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b4/46/93416fdae86d40879714f72956ac14df9c7b76f7d41a4d68aa9f71a0028b/pydantic_settings-2.7.1-py3-none-any.whl", hash = "sha256:590be9e6e24d06db33a4262829edef682500ef008565a969c73d39d5f8bfb3fd", size = 29718 }, +] + [[package]] name = "pyflakes" version = "3.2.0" From 4a592306041b6b587aebbdfdb27dbb0dcf691611 Mon Sep 17 00:00:00 2001 From: Carol Jung Date: Fri, 7 Feb 2025 19:57:51 -0800 Subject: [PATCH 2/5] wip: working on set --- src/codegen/cli/commands/config/main.py | 27 ++++++++++++++++--------- src/codegen/shared/configs/config.py | 2 +- src/codegen/shared/configs/models.py | 25 +++++++++++++++++++++-- 3 files changed, 42 insertions(+), 12 deletions(-) diff --git a/src/codegen/cli/commands/config/main.py b/src/codegen/cli/commands/config/main.py index 9ba19d6ec..13020e068 100644 --- a/src/codegen/cli/commands/config/main.py +++ b/src/codegen/cli/commands/config/main.py @@ -4,6 +4,8 @@ import rich_click as click from rich.table import Table +from codegen.shared.configs.config import config + @click.group(name="config") def config_command(): @@ -14,10 +16,8 @@ def config_command(): @config_command.command(name="list") def list_command(): """List current configuration values.""" - from codegen.shared.configs.config import config - table = Table(title="Configuration Values", border_style="blue", show_header=True) - table.add_column("Key", style="cyan") + table.add_column("Key", style="cyan", no_wrap=True) table.add_column("Value", style="magenta") def flatten_dict(data: dict, prefix: str = "") -> dict: @@ -40,7 +40,7 @@ def get_prefix(item): for prefix, group in groupby(sorted_items, key=get_prefix): table.add_section() - table.add_row(f"[bold blue]{prefix}[/bold blue]", "") + table.add_row(f"[bold yellow]{prefix}[/bold yellow]", "") for key, value in group: # Remove the prefix from the displayed key display_key = key[len(prefix) + 1 :] if "." in key else key @@ -53,14 +53,23 @@ def get_prefix(item): @click.argument("key") def get_command(key: str): """Get a configuration value.""" - # TODO: Implement configuration get logic - rich.print(f"[yellow]Getting configuration value for: {key}[/yellow]") + value = config.get(key) + if value is None: + rich.print(f"[red]Error: Configuration key '{key}' not found[/red]") + return + + rich.print(f"[cyan]{key}[/cyan] = [magenta]{value}[/magenta]") @config_command.command(name="set") @click.argument("key") @click.argument("value") def set_command(key: str, value: str): - """Set a configuration value.""" - # TODO: Implement configuration set logic - rich.print(f"[green]Setting {key}={value}[/green]") + """Set a configuration value and write to config.toml.""" + value = config.get(key) + if value is None: + rich.print(f"[red]Error: Configuration key '{key}' not found[/red]") + return + + config.set(key, value) + rich.print(f"[green]Successfully set {key}=[magenta]{value}[/magenta] and saved to config.toml[/green]") diff --git a/src/codegen/shared/configs/config.py b/src/codegen/shared/configs/config.py index 6e433faa1..29823a6e9 100644 --- a/src/codegen/shared/configs/config.py +++ b/src/codegen/shared/configs/config.py @@ -7,7 +7,7 @@ def load(config_path: Path | None = None) -> Config: - """Load configuration from the config file.""" + """Loads configuration from various sources.""" # Load from .env file env_config = _load_from_env() diff --git a/src/codegen/shared/configs/models.py b/src/codegen/shared/configs/models.py index c8f89fbc2..419ed4fde 100644 --- a/src/codegen/shared/configs/models.py +++ b/src/codegen/shared/configs/models.py @@ -52,7 +52,6 @@ class Config(BaseSettings): repository: RepositoryConfig = Field(default_factory=RepositoryConfig) feature_flags: FeatureFlagsConfig = Field(default_factory=FeatureFlagsConfig) - @staticmethod def save(self, config_path: Path | None = None) -> None: """Save configuration to the config file.""" path = config_path or CONFIG_PATH @@ -60,7 +59,29 @@ def save(self, config_path: Path | None = None) -> None: path.parent.mkdir(parents=True, exist_ok=True) with open(path, "w") as f: - toml.dump(Config.model_dump(exclude_none=True), f) + toml.dump(self.model_dump(exclude_none=True), f) + + def get(self, full_key: str) -> str | None: + data = self.model_dump() + keys = full_key.split(".") + current = data + for k in keys: + if not isinstance(current, dict) or k not in current: + return None + current = current[k] + return current + + def set(self, full_key: str, value: str) -> None: + data = self.model_dump() + keys = full_key.split(".") + current = data + for k in keys[:-1]: + if k not in current: + current[k] = {} + current = current[k] + current[keys[-1]] = value + self.model_validate(data) + self.save() def __str__(self) -> str: """Return a pretty-printed string representation of the config.""" From c76de6ffd9662de29a47db991a0c3880b8743ffd Mon Sep 17 00:00:00 2001 From: Carol Jung Date: Mon, 10 Feb 2025 10:05:54 -0800 Subject: [PATCH 3/5] wip --- .codegen/config.toml | 5 +- src/codegen/cli/commands/config/main.py | 8 +++- src/codegen/sdk/codebase/config.py | 2 +- src/codegen/shared/configs/models.py | 61 +++++++++++++++++-------- 4 files changed, 52 insertions(+), 24 deletions(-) diff --git a/.codegen/config.toml b/.codegen/config.toml index f5c43ba32..7320c344f 100644 --- a/.codegen/config.toml +++ b/.codegen/config.toml @@ -12,14 +12,15 @@ syntax_highlight_enabled = false [feature_flags.codebase] debug = false verify_graph = false -track_graph = true +track_graph = false method_usages = true sync_enabled = true full_range_index = false ignore_process_errors = true disable_graph = false generics = true -import_resolution_overrides = {} + +[feature_flags.codebase.import_resolution_overrides] [feature_flags.codebase.typescript] ts_dependency_manager = false diff --git a/src/codegen/cli/commands/config/main.py b/src/codegen/cli/commands/config/main.py index 13020e068..22772a42b 100644 --- a/src/codegen/cli/commands/config/main.py +++ b/src/codegen/cli/commands/config/main.py @@ -66,10 +66,14 @@ def get_command(key: str): @click.argument("value") def set_command(key: str, value: str): """Set a configuration value and write to config.toml.""" - value = config.get(key) - if value is None: + cur_value = config.get(key) + if cur_value is None: rich.print(f"[red]Error: Configuration key '{key}' not found[/red]") return + if cur_value.lower() == value.lower(): + rich.print(f"[yellow]Warning: Configuration key '{key}' already set to '{value}'[/yellow]") + return + config.set(key, value) rich.print(f"[green]Successfully set {key}=[magenta]{value}[/magenta] and saved to config.toml[/green]") diff --git a/src/codegen/sdk/codebase/config.py b/src/codegen/sdk/codebase/config.py index 0997c7477..780c4fec1 100644 --- a/src/codegen/sdk/codebase/config.py +++ b/src/codegen/sdk/codebase/config.py @@ -35,7 +35,7 @@ class GSFeatureFlags(BaseModel): model_config = ConfigDict(frozen=True) debug: bool = False verify_graph: bool = False - track_graph: bool = True # Track the initial graph state + track_graph: bool = False # Track the initial graph state method_usages: bool = True sync_enabled: bool = True ts_dependency_manager: bool = False # Enable Typescript Dependency Manager diff --git a/src/codegen/shared/configs/models.py b/src/codegen/shared/configs/models.py index 419ed4fde..2914ab0ef 100644 --- a/src/codegen/shared/configs/models.py +++ b/src/codegen/shared/configs/models.py @@ -1,3 +1,4 @@ +import json from pathlib import Path import toml @@ -8,21 +9,21 @@ class TypescriptConfig(BaseModel): - ts_dependency_manager: bool | None = None - ts_language_engine: bool | None = None - v8_ts_engine: bool | None = None + ts_dependency_manager: bool = False + ts_language_engine: bool = False + v8_ts_engine: bool = False class CodebaseFeatureFlags(BaseModel): - debug: bool | None = None - verify_graph: bool | None = None - track_graph: bool | None = None - method_usages: bool | None = None - sync_enabled: bool | None = None - full_range_index: bool | None = None - ignore_process_errors: bool | None = None - disable_graph: bool | None = None - generics: bool | None = None + debug: bool = False + verify_graph: bool = False + track_graph: bool = False + method_usages: bool = True + sync_enabled: bool = True + full_range_index: bool = False + ignore_process_errors: bool = True + disable_graph: bool = False + generics: bool = True import_resolution_overrides: dict[str, str] = Field(default_factory=lambda: {}) typescript: TypescriptConfig = Field(default_factory=TypescriptConfig) @@ -62,6 +63,7 @@ def save(self, config_path: Path | None = None) -> None: toml.dump(self.model_dump(exclude_none=True), f) def get(self, full_key: str) -> str | None: + """Get a configuration value as a JSON string.""" data = self.model_dump() keys = full_key.split(".") current = data @@ -69,22 +71,43 @@ def get(self, full_key: str) -> str | None: if not isinstance(current, dict) or k not in current: return None current = current[k] - return current + return json.dumps(current) def set(self, full_key: str, value: str) -> None: + """Update a configuration value and save it to the config file. + + Args: + full_key: Dot-separated path to the config value (e.g. "feature_flags.codebase.debug") + value: string representing the new value + """ + # Navigate to the correct nested dictionary data = self.model_dump() keys = full_key.split(".") current = data + + # Traverse until the second-to-last key for k in keys[:-1]: - if k not in current: - current[k] = {} + if not isinstance(current, dict) or k not in current: + msg = f"Invalid configuration path: {full_key}" + raise KeyError(msg) current = current[k] - current[keys[-1]] = value - self.model_validate(data) + + # Set the value at the final key + if not isinstance(current, dict) or keys[-1] not in current: + msg = f"Invalid configuration path: {full_key}" + raise KeyError(msg) + + if isinstance(current[keys[-1]], dict): + current[keys[-1]] = json.loads(value) + else: + current[keys[-1]] = value + + # Update the Config object with the new data + self.__dict__.update(self.__class__.model_validate(data).__dict__) + + # Save to config file self.save() def __str__(self) -> str: """Return a pretty-printed string representation of the config.""" - import json - return json.dumps(self.model_dump(exclude_none=False), indent=2) From 03b98b63bf2e5c7a3b35fd4afb1a5ff5e044853a Mon Sep 17 00:00:00 2001 From: Carol Jung Date: Mon, 10 Feb 2025 12:21:57 -0800 Subject: [PATCH 4/5] fix --- .codegen/config.toml | 21 ------------------- src/codegen/cli/commands/config/main.py | 15 ++++++++++---- src/codegen/sdk/codebase/config.py | 2 +- src/codegen/shared/configs/config.py | 3 ++- src/codegen/shared/configs/models.py | 27 ++++++++++++++++++++----- 5 files changed, 36 insertions(+), 32 deletions(-) diff --git a/.codegen/config.toml b/.codegen/config.toml index 7320c344f..c2ff34531 100644 --- a/.codegen/config.toml +++ b/.codegen/config.toml @@ -5,24 +5,3 @@ openai_api_key = "" [repository] organization_name = "codegen-sh" repo_name = "codegen-sdk" - -[feature_flags] -syntax_highlight_enabled = false - -[feature_flags.codebase] -debug = false -verify_graph = false -track_graph = false -method_usages = true -sync_enabled = true -full_range_index = false -ignore_process_errors = true -disable_graph = false -generics = true - -[feature_flags.codebase.import_resolution_overrides] - -[feature_flags.codebase.typescript] -ts_dependency_manager = false -ts_language_engine = false -v8_ts_engine = false diff --git a/src/codegen/cli/commands/config/main.py b/src/codegen/cli/commands/config/main.py index 22772a42b..05fbe8f2c 100644 --- a/src/codegen/cli/commands/config/main.py +++ b/src/codegen/cli/commands/config/main.py @@ -1,3 +1,4 @@ +import logging from itertools import groupby import rich @@ -25,6 +26,9 @@ def flatten_dict(data: dict, prefix: str = "") -> dict: for key, value in data.items(): full_key = f"{prefix}{key}" if prefix else key if isinstance(value, dict): + # Always include dictionary fields, even if empty + if not value: + items[full_key] = "{}" items.update(flatten_dict(value, f"{full_key}.")) else: items[full_key] = value @@ -71,9 +75,12 @@ def set_command(key: str, value: str): rich.print(f"[red]Error: Configuration key '{key}' not found[/red]") return - if cur_value.lower() == value.lower(): - rich.print(f"[yellow]Warning: Configuration key '{key}' already set to '{value}'[/yellow]") - return + if cur_value.lower() != value.lower(): + try: + config.set(key, value) + except Exception as e: + logging.exception(e) + rich.print(f"[red]{e}[/red]") + return - config.set(key, value) rich.print(f"[green]Successfully set {key}=[magenta]{value}[/magenta] and saved to config.toml[/green]") diff --git a/src/codegen/sdk/codebase/config.py b/src/codegen/sdk/codebase/config.py index 780c4fec1..78a5219cf 100644 --- a/src/codegen/sdk/codebase/config.py +++ b/src/codegen/sdk/codebase/config.py @@ -50,7 +50,7 @@ class GSFeatureFlags(BaseModel): DefaultFlags = GSFeatureFlags(sync_enabled=False) -TestFlags = GSFeatureFlags(debug=True, verify_graph=True, full_range_index=True) +TestFlags = GSFeatureFlags(debug=True, track_graph=True, verify_graph=True, full_range_index=True) LintFlags = GSFeatureFlags(method_usages=False) ParseTestFlags = GSFeatureFlags(debug=False, track_graph=False) diff --git a/src/codegen/shared/configs/config.py b/src/codegen/shared/configs/config.py index 29823a6e9..3bf6382d6 100644 --- a/src/codegen/shared/configs/config.py +++ b/src/codegen/shared/configs/config.py @@ -30,7 +30,8 @@ def _load_from_toml(config_path: Path) -> Config: if config_path.exists(): with open(config_path, "rb") as f: toml_config = tomllib.load(f) - return Config.model_validate(toml_config) + return Config.model_validate(toml_config, strict=False) + return Config() diff --git a/src/codegen/shared/configs/models.py b/src/codegen/shared/configs/models.py index 2914ab0ef..474665768 100644 --- a/src/codegen/shared/configs/models.py +++ b/src/codegen/shared/configs/models.py @@ -44,11 +44,14 @@ class SecretsConfig(BaseSettings): class FeatureFlagsConfig(BaseModel): - syntax_highlight_enabled: bool | None = None codebase: CodebaseFeatureFlags = Field(default_factory=CodebaseFeatureFlags) class Config(BaseSettings): + model_config = SettingsConfigDict( + extra="ignore", + exclude_defaults=False, + ) secrets: SecretsConfig = Field(default_factory=SecretsConfig) repository: RepositoryConfig = Field(default_factory=RepositoryConfig) feature_flags: FeatureFlagsConfig = Field(default_factory=FeatureFlagsConfig) @@ -80,25 +83,39 @@ def set(self, full_key: str, value: str) -> None: full_key: Dot-separated path to the config value (e.g. "feature_flags.codebase.debug") value: string representing the new value """ - # Navigate to the correct nested dictionary data = self.model_dump() keys = full_key.split(".") current = data + current_attr = self - # Traverse until the second-to-last key + # Traverse through the key path and validate for k in keys[:-1]: if not isinstance(current, dict) or k not in current: msg = f"Invalid configuration path: {full_key}" raise KeyError(msg) current = current[k] + current_attr = current_attr.__getattribute__(k) - # Set the value at the final key if not isinstance(current, dict) or keys[-1] not in current: msg = f"Invalid configuration path: {full_key}" raise KeyError(msg) + # Validate the value type at key + field_info = current_attr.model_fields[keys[-1]].annotation + if isinstance(field_info, BaseModel): + try: + Config.model_validate(value, strict=False) + except Exception as e: + msg = f"Value does not match the expected type for key: {full_key}\n\nError:{e}" + raise ValueError(msg) + + # Set the key value if isinstance(current[keys[-1]], dict): - current[keys[-1]] = json.loads(value) + try: + current[keys[-1]] = json.loads(value) + except json.JSONDecodeError as e: + msg = f"Value must be a valid JSON object for key: {full_key}\n\nError:{e}" + raise ValueError(msg) else: current[keys[-1]] = value From 858a9aa8007c4bce61de83d5696b659045a3f123 Mon Sep 17 00:00:00 2001 From: Carol Jung Date: Mon, 10 Feb 2025 12:43:45 -0800 Subject: [PATCH 5/5] unit tests --- .codegen/config.toml | 18 +++ src/codegen/shared/configs/models.py | 24 ++-- .../codegen/cli/{ => commands}/conftest.py | 0 .../codegen/cli/{ => commands}/test_reset.py | 0 tests/shared/configs/sample_config.py | 59 ++++++++++ tests/unit/codegen/shared/configs/conftest.py | 42 +++++++ .../codegen/shared/configs/test_config.py | 107 ++++++++++++++++++ .../codegen/shared/configs/test_constants.py | 30 +++++ .../codegen/shared/configs/test_models.py | 105 +++++++++++++++++ 9 files changed, 373 insertions(+), 12 deletions(-) rename tests/integration/codegen/cli/{ => commands}/conftest.py (100%) rename tests/integration/codegen/cli/{ => commands}/test_reset.py (100%) create mode 100644 tests/shared/configs/sample_config.py create mode 100644 tests/unit/codegen/shared/configs/conftest.py create mode 100644 tests/unit/codegen/shared/configs/test_config.py create mode 100644 tests/unit/codegen/shared/configs/test_constants.py create mode 100644 tests/unit/codegen/shared/configs/test_models.py diff --git a/.codegen/config.toml b/.codegen/config.toml index c2ff34531..c8b8658f8 100644 --- a/.codegen/config.toml +++ b/.codegen/config.toml @@ -5,3 +5,21 @@ openai_api_key = "" [repository] organization_name = "codegen-sh" repo_name = "codegen-sdk" + +[feature_flags.codebase] +debug = false +verify_graph = false +track_graph = false +method_usages = true +sync_enabled = true +full_range_index = false +ignore_process_errors = true +disable_graph = false +generics = true + +[feature_flags.codebase.import_resolution_overrides] + +[feature_flags.codebase.typescript] +ts_dependency_manager = false +ts_language_engine = false +v8_ts_engine = false diff --git a/src/codegen/shared/configs/models.py b/src/codegen/shared/configs/models.py index 474665768..30b108b95 100644 --- a/src/codegen/shared/configs/models.py +++ b/src/codegen/shared/configs/models.py @@ -9,21 +9,21 @@ class TypescriptConfig(BaseModel): - ts_dependency_manager: bool = False - ts_language_engine: bool = False - v8_ts_engine: bool = False + ts_dependency_manager: bool | None = None + ts_language_engine: bool | None = None + v8_ts_engine: bool | None = None class CodebaseFeatureFlags(BaseModel): - debug: bool = False - verify_graph: bool = False - track_graph: bool = False - method_usages: bool = True - sync_enabled: bool = True - full_range_index: bool = False - ignore_process_errors: bool = True - disable_graph: bool = False - generics: bool = True + debug: bool | None = None + verify_graph: bool | None = None + track_graph: bool | None = None + method_usages: bool | None = None + sync_enabled: bool | None = None + full_range_index: bool | None = None + ignore_process_errors: bool | None = None + disable_graph: bool | None = None + generics: bool | None = None import_resolution_overrides: dict[str, str] = Field(default_factory=lambda: {}) typescript: TypescriptConfig = Field(default_factory=TypescriptConfig) diff --git a/tests/integration/codegen/cli/conftest.py b/tests/integration/codegen/cli/commands/conftest.py similarity index 100% rename from tests/integration/codegen/cli/conftest.py rename to tests/integration/codegen/cli/commands/conftest.py diff --git a/tests/integration/codegen/cli/test_reset.py b/tests/integration/codegen/cli/commands/test_reset.py similarity index 100% rename from tests/integration/codegen/cli/test_reset.py rename to tests/integration/codegen/cli/commands/test_reset.py diff --git a/tests/shared/configs/sample_config.py b/tests/shared/configs/sample_config.py new file mode 100644 index 000000000..b3a5b0ced --- /dev/null +++ b/tests/shared/configs/sample_config.py @@ -0,0 +1,59 @@ +# Test data +SAMPLE_TOML = """ +[secrets] +github_token = "gh_token123" +openai_api_key = "sk-123456" + +[repository] +organization_name = "test-org" +repo_name = "test-repo" + +[feature_flags.codebase] +debug = true +verify_graph = true +track_graph = false +method_usages = true +sync_enabled = true +full_range_index = false +ignore_process_errors = true +disable_graph = false +generics = true + +[feature_flags.codebase.typescript] +ts_dependency_manager = true +ts_language_engine = false +v8_ts_engine = true + +[feature_flags.codebase.import_resolution_overrides] +"@org/pkg" = "./local/path" +""" + +SAMPLE_CONFIG_DICT = { + "secrets": { + "github_token": "gh_token123", + "openai_api_key": "sk-123456", + }, + "repository": { + "organization_name": "test-org", + "repo_name": "test-repo", + }, + "feature_flags": { + "codebase": { + "debug": True, + "verify_graph": True, + "track_graph": False, + "method_usages": True, + "sync_enabled": True, + "full_range_index": False, + "ignore_process_errors": True, + "disable_graph": False, + "generics": True, + "typescript": { + "ts_dependency_manager": True, + "ts_language_engine": False, + "v8_ts_engine": True, + }, + "import_resolution_overrides": {"@org/pkg": "./local/path"}, + } + }, +} diff --git a/tests/unit/codegen/shared/configs/conftest.py b/tests/unit/codegen/shared/configs/conftest.py new file mode 100644 index 000000000..d6a7304fa --- /dev/null +++ b/tests/unit/codegen/shared/configs/conftest.py @@ -0,0 +1,42 @@ +from unittest.mock import patch + +import pytest + +from tests.shared.configs.sample_config import SAMPLE_CONFIG_DICT, SAMPLE_TOML + + +@pytest.fixture +def sample_toml(): + """Return sample TOML configuration string.""" + return SAMPLE_TOML + + +@pytest.fixture +def sample_config_dict(): + """Return sample configuration dictionary.""" + return SAMPLE_CONFIG_DICT + + +@pytest.fixture +def temp_config_file(tmp_path): + """Create a temporary config file with sample TOML content.""" + config_file = tmp_path / "config.toml" + config_file.write_text(SAMPLE_TOML) + return config_file + + +@pytest.fixture +def invalid_toml_file(tmp_path): + """Create a temporary file with invalid TOML content.""" + invalid_toml = tmp_path / "invalid.toml" + invalid_toml.write_text("invalid = toml [ content") + return invalid_toml + + +@pytest.fixture +def clean_env(): + """Temporarily clear environment variables and override env file path.""" + with patch.dict("os.environ", {}, clear=True): + with patch("codegen.shared.configs.models.Config.model_config", {"env_file": "nonexistent.env"}): + with patch("codegen.shared.configs.models.SecretsConfig.model_config", {"env_file": "nonexistent.env"}): + yield diff --git a/tests/unit/codegen/shared/configs/test_config.py b/tests/unit/codegen/shared/configs/test_config.py new file mode 100644 index 000000000..1a195b561 --- /dev/null +++ b/tests/unit/codegen/shared/configs/test_config.py @@ -0,0 +1,107 @@ +from pathlib import Path +from unittest.mock import patch + +import pytest +import tomllib + +from codegen.shared.configs.config import ( + Config, + _load_from_env, + _load_from_toml, + _merge_configs, + load, +) +from codegen.shared.configs.models import CodebaseFeatureFlags, FeatureFlagsConfig, SecretsConfig + + +# Test _merge_configs +def test_merge_configs_basic(): + base = {"a": 1, "b": 2} + override = {"b": 3, "c": 4} + result = _merge_configs(base, override) + assert result == {"a": 1, "b": 3, "c": 4} + + +def test_merge_configs_nested(): + base = {"feature_flags": {"codebase": {"debug": False, "typescript": {"ts_dependency_manager": False}}}} + override = {"feature_flags": {"codebase": {"debug": True, "typescript": {"ts_language_engine": True}}}} + result = _merge_configs(base, override) + assert result == {"feature_flags": {"codebase": {"debug": True, "typescript": {"ts_dependency_manager": False, "ts_language_engine": True}}}} + + +def test_merge_configs_none_values(): + base = {"secrets": {"github_token": "token1"}} + override = {"secrets": {"github_token": None}} + result = _merge_configs(base, override) + assert result == {"secrets": {"github_token": "token1"}} + + +def test_merge_configs_empty_string(): + base = {"repository": {"organization_name": "org1"}} + override = {"repository": {"organization_name": ""}} + result = _merge_configs(base, override) + assert result == {"repository": {"organization_name": "org1"}} + + +# Test _load_from_toml +def test_load_from_toml_existing_file(temp_config_file): + config = _load_from_toml(temp_config_file) + assert isinstance(config, Config) + assert config.secrets.github_token == "gh_token123" + assert config.repository.organization_name == "test-org" + assert config.feature_flags.codebase.debug is True + assert config.feature_flags.codebase.typescript.ts_dependency_manager is True + assert config.feature_flags.codebase.import_resolution_overrides == {"@org/pkg": "./local/path"} + + +@patch("codegen.shared.configs.models.SecretsConfig.model_config", {"env_file": "nonexistent.env"}) +def test_load_from_toml_nonexistent_file(): + config = _load_from_toml(Path("nonexistent.toml")) + assert isinstance(config, Config) + assert config.secrets.github_token is None + assert config.repository.organization_name is None + assert config.feature_flags.codebase.debug is None + + +# Test _load_from_env +@patch.dict("os.environ", {"CODEGEN_SECRETS__GITHUB_TOKEN": "env_token", "CODEGEN_SECRETS__OPENAI_API_KEY": "env_key"}) +def test_load_from_env(): + config = _load_from_env() + assert isinstance(config, Config) + assert config.secrets.github_token == "env_token" + assert config.secrets.openai_api_key == "env_key" + + +# Test load function +@patch.dict("os.environ", {}, clear=True) # Clear all env vars for this test +@patch("codegen.shared.configs.config._load_from_env") +@patch("codegen.shared.configs.config._load_from_toml") +@patch("codegen.shared.configs.models.SecretsConfig.model_config", {"env_file": None, "env_prefix": "CODEGEN_SECRETS__"}) +def test_load_with_both_configs(mock_toml, mock_env): + # Setup mock returns + mock_env.return_value = Config(secrets=SecretsConfig(github_token="env_token"), feature_flags=FeatureFlagsConfig(codebase=CodebaseFeatureFlags(debug=True))) + mock_toml.return_value = Config(secrets={"openai_api_key": "openai_key"}, repository={"organization_name": "codegen-org"}) + + config = load() + + assert isinstance(config, Config) + assert config.secrets.github_token == "env_token" + assert config.secrets.openai_api_key == "openai_key" + assert config.repository.organization_name == "codegen-org" + assert config.feature_flags.codebase.debug is True + + +@patch("codegen.shared.configs.config._load_from_env") +@patch("codegen.shared.configs.config._load_from_toml") +def test_load_with_custom_path(mock_toml, mock_env): + custom_path = Path("custom/config.toml") + load(config_path=custom_path) + + mock_toml.assert_called_once_with(custom_path) + mock_env.assert_called_once() + + +# Error cases +def test_load_from_toml_invalid_file(invalid_toml_file): + with pytest.raises(tomllib.TOMLDecodeError): + _load_from_toml(invalid_toml_file) diff --git a/tests/unit/codegen/shared/configs/test_constants.py b/tests/unit/codegen/shared/configs/test_constants.py new file mode 100644 index 000000000..dc18c703a --- /dev/null +++ b/tests/unit/codegen/shared/configs/test_constants.py @@ -0,0 +1,30 @@ +from pathlib import Path + +from codegen.shared.configs.constants import ( + CODEGEN_DIR_NAME, + CODEGEN_REPO_ROOT, + CONFIG_FILENAME, + CONFIG_PATH, + ENV_FILENAME, + ENV_PATH, +) + + +def test_codegen_repo_root_is_path(): + assert isinstance(CODEGEN_REPO_ROOT, Path) + assert CODEGEN_REPO_ROOT.exists() + assert CODEGEN_REPO_ROOT.is_dir() + + +def test_config_path_construction(): + expected_path = CODEGEN_REPO_ROOT / CODEGEN_DIR_NAME / CONFIG_FILENAME + assert CONFIG_PATH == expected_path + assert str(CONFIG_PATH).endswith(f"{CODEGEN_DIR_NAME}/{CONFIG_FILENAME}") + assert CONFIG_PATH.exists() + assert CONFIG_PATH.is_file() + + +def test_env_path_construction(): + expected_path = CODEGEN_REPO_ROOT / "src" / "codegen" / ENV_FILENAME + assert ENV_PATH == expected_path + assert str(ENV_PATH).endswith(f"src/codegen/{ENV_FILENAME}") diff --git a/tests/unit/codegen/shared/configs/test_models.py b/tests/unit/codegen/shared/configs/test_models.py new file mode 100644 index 000000000..aa4e3e498 --- /dev/null +++ b/tests/unit/codegen/shared/configs/test_models.py @@ -0,0 +1,105 @@ +import json +from pathlib import Path +from unittest.mock import mock_open, patch + +import pytest +import toml + +from codegen.shared.configs.models import CodebaseFeatureFlags, Config, FeatureFlagsConfig, RepositoryConfig + + +@pytest.fixture +def sample_config(): + codebase_flags = CodebaseFeatureFlags(debug=True, verify_graph=False) + return Config(repository=RepositoryConfig(organization_name="test-org", repo_name="test-repo"), feature_flags=FeatureFlagsConfig(codebase=codebase_flags)) + + +def test_config_initialization(): + config = Config() + assert config.repository is not None + assert config.feature_flags is not None + assert config.secrets is not None + + +def test_config_with_values(): + config = Config(repository={"organization_name": "test-org", "repo_name": "test-repo"}) + assert config.repository.organization_name == "test-org" + assert config.repository.repo_name == "test-repo" + + +@patch("builtins.open", new_callable=mock_open) +@patch("pathlib.Path.mkdir") +def test_save_config(mock_mkdir, mock_file, sample_config): + sample_config.save(Path("test_config.toml")) + + mock_mkdir.assert_called_once_with(parents=True, exist_ok=True) + mock_file.assert_called_once_with(Path("test_config.toml"), "w") + + # Verify the content being written + written_data = mock_file().write.call_args[0][0] + parsed_data = toml.loads(written_data) + assert parsed_data["repository"]["organization_name"] == "test-org" + + +def test_get_config_value(sample_config): + # Test getting a simple value + assert json.loads(sample_config.get("repository.organization_name")) == "test-org" + + # Test getting a nested value + assert json.loads(sample_config.get("feature_flags.codebase.debug")) is True + + # Test getting non-existent value + assert sample_config.get("invalid.path") is None + + +def test_set_config_value(sample_config): + # Instead of mocking save, we'll mock the open function used within save + with patch("builtins.open", new_callable=mock_open) as mock_file: + # Test setting a simple string value + sample_config.set("repository.organization_name", "new-org") + assert sample_config.repository.organization_name == "new-org" + + # Test setting a boolean value + sample_config.set("feature_flags.codebase.debug", "false") + assert not sample_config.feature_flags.codebase.debug + + # Verify save was called by checking if open was called + assert mock_file.called + + +def test_set_config_invalid_path(sample_config): + with pytest.raises(KeyError, match="Invalid configuration path: invalid.path"): + sample_config.set("invalid.path", "value") + + +def test_set_config_invalid_json(sample_config): + with pytest.raises(ValueError, match="Value must be a valid JSON object"): + sample_config.set("repository", "invalid json {") + + +def test_config_str_representation(sample_config): + config_str = str(sample_config) + assert isinstance(config_str, str) + # Verify it's valid JSON + parsed = json.loads(config_str) + assert parsed["repository"]["organization_name"] == "test-org" + + +def test_set_config_new_override_key(sample_config): + with patch("builtins.open", new_callable=mock_open) as mock_file: + # Test setting a new import resolution override + sample_config.set("feature_flags.codebase.import_resolution_overrides", '{"new_key": "new_value"}') + + # Verify the new key was added + assert sample_config.feature_flags.codebase.import_resolution_overrides["new_key"] == "new_value" + + # Verify save was called + assert mock_file.called + + # Test adding another key to the existing overrides + sample_config.set("feature_flags.codebase.import_resolution_overrides", '{"new_key": "new_value", "another_key": "another_value"}') + + # Verify both keys exist + overrides = sample_config.feature_flags.codebase.import_resolution_overrides + assert overrides["new_key"] == "new_value" + assert overrides["another_key"] == "another_value"