From 67bdc6cc68039ea806ed0c1e95aa121faf77e4e7 Mon Sep 17 00:00:00 2001 From: David Turecek Date: Mon, 13 Jul 2026 09:42:47 +0200 Subject: [PATCH] feat(assessors): add lint config coverage assessor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Binary lint detection doesn't distinguish a formatting-only config from one running correctness, standards, and security tools — both score the same. Agents generating code in a formatting-only repo get no automated signal about logic errors or security anti-patterns. New LintConfigCoverageAssessor (lint_config_coverage, T2, 2%) parses lint configs and CI workflows to categorize detected tools into three buckets (correctness, standards, security), scoring 33/67/100 for 1/2/3 categories covered. Supports Go (.golangci.yml), Python (pyproject.toml, ruff.toml, .flake8, etc.), and JS/TS (ESLint, tsconfig). CI workflow files are also scanned. Closes #511 Co-authored-by: Cursor --- docs/attributes.md | 99 +++ src/agentready/assessors/__init__.py | 4 +- src/agentready/assessors/code_quality.py | 791 ++++++++++++++++++++++ src/agentready/data/default-weights.yaml | 16 +- tests/unit/test_assessors_code_quality.py | 416 ++++++++++++ uv.lock | 2 +- 6 files changed, 1321 insertions(+), 7 deletions(-) diff --git a/docs/attributes.md b/docs/attributes.md index 431a9746..08a15883 100644 --- a/docs/attributes.md +++ b/docs/attributes.md @@ -861,6 +861,105 @@ pre-commit run --all-files --- +### 10b. Lint Config Coverage + +**ID**: `lint_config_coverage` +**Weight**: 2% +**Category**: Code Quality +**Status**: ✅ Implemented + +#### Definition + +Evaluates whether the repository's lint tooling covers all three essential categories: **correctness** (logic errors and type violations), **standards** (style and convention enforcement), and **security** (vulnerability pattern detection). A repo that only runs a formatter is not meaningfully safer for AI-assisted development than one with no lint at all. + +#### Why It Matters + +Agents generate code that compiles and passes formatting checks but may still silently drop errors, mishandle types, or introduce known security anti-patterns. Lint tooling that covers all three categories gives the agent automated, deterministic signal about each class of issue. Without security linting, an agent generating Go has no automated feedback on unhandled errors or injection risks. + +#### Measurable Criteria + +Score is proportional to the number of lint categories covered: + +| Categories covered | Score | Status | +|--------------------|-------|--------| +| 0/3 | 0 | fail | +| 1/3 | 33 | fail | +| 2/3 | 67 | fail | +| 3/3 | 100 | **pass** | + +**Sources checked**: standalone lint config files, `.pre-commit-config.yaml` hook IDs, and CI workflow files (`.github/workflows/`, `.gitlab-ci.yml`, `.circleci/config.yml`, `.travis.yml`). + +#### Tool Catalog + +**Go** (parsed from `.golangci.yml`/`.golangci.yaml`/`.golangci.toml`): + +| Category | Tools | +|----------|-------| +| Correctness | `errcheck`, `staticcheck`, `gosimple`, `ineffassign`, `typecheck`, `govet`, `unused`, `nilerr`, `errorlint`, `nilnil`, `exhaustive`, `unparam`, `funclen`, `gocritic` (also standards) | +| Standards | `revive`, `stylecheck`, `gofmt`, `goimports`, `godot`, `misspell`, `whitespace`, `wsl`, `lll`, `maintidx`, `godox`, `gochecknoinits`, `gocritic` (also correctness) | +| Security | `gosec`, `bodyclose`, `sqlclosecheck`, `rowserrcheck`, `noctx` | + +**Python** (parsed from `pyproject.toml [tool.*]`, `setup.cfg`, `.flake8`, `ruff.toml`, `mypy.ini`, `.mypy.ini`, `pyrightconfig.json`, `.pylintrc`, `.bandit`, `.pre-commit-config.yaml`): + +| Category | Tools | +|----------|-------| +| Correctness | `mypy`, `pyright`, `pyflakes`, `pylint` (also standards), `ruff` (also standards), `flake8` (also standards) | +| Standards | `ruff`, `flake8`, `pylint`, `black`, `isort`, `autopep8`, `pydocstyle`, `pycodestyle` | +| Security | `bandit`, `semgrep`, `safety`, `pip-audit`, `dlint`, `flake8-bandit` | + +**JavaScript/TypeScript** (parsed from `.eslintrc.*`, `.eslintrc.yaml`, `eslint.config.*`, `package.json eslintConfig`, `tsconfig.json`, `.prettierrc*`): + +| Category | Tools / Configs | +|----------|-----------------| +| Correctness | `eslint:recommended` (also standards), `eslint:all` (also standards), `@typescript-eslint`, `typescript-eslint`, `@typescript-eslint/recommended`, `@typescript-eslint/recommended-type-checked`, `@typescript-eslint/strict-type-checked`, `tsconfig.json strict: true` | +| Standards | `eslint:recommended`, `eslint:all`, `airbnb`, `airbnb-base`, `standard`, `prettier`, `plugin:import`, `plugin:unicorn` | +| Security | `plugin:security`, `security`, `eslint-plugin-security`, `no-unsanitized`, `no-secrets` | + +#### Remediation + +**Python** — add all three categories: + +```toml +# pyproject.toml +[tool.mypy] +strict = true + +[tool.ruff] +select = ["E", "F", "S"] # E/F = correctness+standards, S = flake8-bandit (security) + +[tool.bandit] +targets = ["src"] +``` + +**Go** — enable tools for all categories in `.golangci.yml`: + +```yaml +linters: + enable: + - errcheck # correctness + - staticcheck # correctness + - revive # standards + - gosec # security +``` + +**JavaScript/TypeScript** — configure ESLint with all three: + +```json +{ + "extends": [ + "eslint:recommended", + "plugin:@typescript-eslint/recommended-type-checked", + "plugin:security/recommended" + ] +} +``` + +**Citations**: + +- agentready: [Issue #511 — Lint assessment: check config coverage](https://github.com/ambient-code/agentready/issues/511) + +--- + ### 11. Conventional Commit Messages **ID**: `conventional_commits` diff --git a/src/agentready/assessors/__init__.py b/src/agentready/assessors/__init__.py index 70bea13d..83572e99 100644 --- a/src/agentready/assessors/__init__.py +++ b/src/agentready/assessors/__init__.py @@ -10,6 +10,7 @@ from .base import BaseAssessor from .code_quality import ( CyclomaticComplexityAssessor, + LintConfigCoverageAssessor, StructuredLoggingAssessor, TypeAnnotationsAssessor, ) @@ -85,7 +86,7 @@ def create_all_assessors() -> list[BaseAssessor]: DependencySecurityAssessor(), # 5% DbtProjectConfigAssessor(), # dbt conditional DbtModelDocumentationAssessor(), # dbt conditional - # Tier 2 Critical — 27% total (9 attributes, 3% each) + # Tier 2 Critical — 27% total (9 attributes at 3% each) + lint_config_coverage, gitignore_completeness, pattern_references (2% each) DeterministicEnforcementAssessor(), ConventionalCommitsAssessor(), GitignoreAssessor(), @@ -95,6 +96,7 @@ def create_all_assessors() -> list[BaseAssessor]: InlineDocumentationAssessor(), PatternReferencesAssessor(), DesignIntentAssessor(), # 3% (moved from T3) + LintConfigCoverageAssessor(), # 2% (issue #511) DbtDataTestsAssessor(), # dbt conditional DbtProjectStructureAssessor(), # dbt conditional # Tier 3 Important — 13% total (8 attributes) diff --git a/src/agentready/assessors/code_quality.py b/src/agentready/assessors/code_quality.py index 0bab5246..898ed335 100644 --- a/src/agentready/assessors/code_quality.py +++ b/src/agentready/assessors/code_quality.py @@ -2,11 +2,13 @@ import ast import configparser +import json import logging import os import re import subprocess import tomllib +from pathlib import Path import lizard import radon.complexity @@ -1134,3 +1136,792 @@ def _create_remediation(self) -> Remediation: ), ], ) + + +class LintConfigCoverageAssessor(BaseAssessor): + """Assesses lint config coverage across correctness, standards, and security categories. + + Tier 2 Critical (2% weight) - A formatting-only lint config gives an agent + no automated signal about correctness errors or security issues in code it + generates. This assessor distinguishes between lint breadth and depth. + + Scoring: + - 0/3 categories: 0 pts (fail) + - 1/3 categories: 33 pts (fail) + - 2/3 categories: 67 pts (fail) + - 3/3 categories: 100 pts (pass) + + Sources checked: standalone lint config files, pre-commit hooks, CI workflow files. + """ + + # Per-language mapping of tool name → set of covered categories. + # A tool covering multiple categories counts for all of them. + _PYTHON_TOOLS: dict[str, set[str]] = { + # correctness + "mypy": {"correctness"}, + "pyright": {"correctness"}, + "pyflakes": {"correctness"}, + "pylint": {"correctness", "standards"}, + "ruff": {"correctness", "standards"}, + "flake8": {"correctness", "standards"}, + # standards only + "black": {"standards"}, + "isort": {"standards"}, + "autopep8": {"standards"}, + "pydocstyle": {"standards"}, + "pycodestyle": {"standards"}, + # security + "bandit": {"security"}, + "semgrep": {"security"}, + "safety": {"security"}, + "pip-audit": {"security"}, + "dlint": {"security"}, + "flake8-bandit": {"security"}, + } + + _GO_TOOLS: dict[str, set[str]] = { + # correctness + "errcheck": {"correctness"}, + "staticcheck": {"correctness"}, + "gosimple": {"correctness"}, + "ineffassign": {"correctness"}, + "typecheck": {"correctness"}, + "govet": {"correctness"}, + "unused": {"correctness"}, + "nilerr": {"correctness"}, + "errorlint": {"correctness"}, + "nilnil": {"correctness"}, + "exhaustive": {"correctness"}, + "unparam": {"correctness"}, + "funclen": {"correctness"}, + # standards + "revive": {"standards"}, + "stylecheck": {"standards"}, + "gofmt": {"standards"}, + "goimports": {"standards"}, + "godot": {"standards"}, + "misspell": {"standards"}, + "whitespace": {"standards"}, + "wsl": {"standards"}, + "lll": {"standards"}, + "maintidx": {"standards"}, + "godox": {"standards"}, + "gochecknoinits": {"standards"}, + # both + "gocritic": {"correctness", "standards"}, + # security + "gosec": {"security"}, + "bodyclose": {"security"}, + "sqlclosecheck": {"security"}, + "rowserrcheck": {"security"}, + "noctx": {"security"}, + } + + _JS_TOOLS: dict[str, set[str]] = { + # correctness — TypeScript-specific + "@typescript-eslint": {"correctness"}, + "typescript-eslint": {"correctness"}, + "@typescript-eslint/recommended": {"correctness"}, + "@typescript-eslint/recommended-type-checked": {"correctness"}, + "@typescript-eslint/strict-type-checked": {"correctness"}, + # correctness + standards (ESLint core rules catch real bugs as well as style) + "eslint:recommended": {"correctness", "standards"}, + "eslint:all": {"correctness", "standards"}, + "airbnb": {"standards"}, + "airbnb-base": {"standards"}, + "standard": {"standards"}, + "prettier": {"standards"}, + "plugin:import": {"standards"}, + "plugin:unicorn": {"standards"}, + # security + "plugin:security": {"security"}, + "security": {"security"}, + "no-unsanitized": {"security"}, + "no-secrets": {"security"}, + "eslint-plugin-security": {"security"}, + } + + CATEGORIES = ("correctness", "standards", "security") + + @property + def attribute_id(self) -> str: + return "lint_config_coverage" + + @property + def tier(self) -> int: + return 2 + + @property + def attribute(self) -> Attribute: + return Attribute( + id=self.attribute_id, + name="Lint Config Coverage", + category="Code Quality", + tier=self.tier, + description="Lint config covers correctness, standards, and security categories", + criteria="All three lint categories present: correctness, standards, security", + default_weight=0.02, + ) + + def is_applicable(self, repository: Repository) -> bool: + supported = {"Python", "Go", "JavaScript", "TypeScript"} + return bool(set(repository.languages.keys()) & supported) + + def assess(self, repository: Repository) -> Finding: + primary = self._primary_language( + repository, {"Python", "Go", "JavaScript", "TypeScript"} + ) + if primary == "Python": + tools = self._collect_python_tools(repository) + catalog = self._PYTHON_TOOLS + elif primary == "Go": + tools = self._collect_go_tools(repository) + catalog = self._GO_TOOLS + elif primary in ("JavaScript", "TypeScript"): + tools = self._collect_js_tools(repository) + catalog = self._JS_TOOLS + else: + return Finding.not_applicable( + self.attribute, + reason=f"Lint coverage check not implemented for {list(repository.languages.keys())}", + ) + + # Merge CI-detected tools (language-agnostic text scan) + ci_tools = self._collect_ci_tools(repository, primary) + tools = tools | ci_tools + + covered: set[str] = set() + tools_by_cat: dict[str, list[str]] = {c: [] for c in self.CATEGORIES} + for tool in tools: + tool_lower = tool.lower() + for catalog_tool, cats in catalog.items(): + if tool_lower == catalog_tool: + for cat in cats: + covered.add(cat) + tools_by_cat[cat].append(tool) + + n_covered = len(covered) + score = self.calculate_proportional_score(n_covered, 3) + + evidence = self._build_evidence(covered, tools_by_cat, tools) + + if n_covered == 3: + return Finding( + attribute=self.attribute, + status="pass", + score=score, + measured_value="3/3 categories", + threshold="all 3 categories (correctness, standards, security)", + evidence=evidence, + remediation=None, + error_message=None, + ) + + missing = [c for c in self.CATEGORIES if c not in covered] + return Finding( + attribute=self.attribute, + status="fail", + score=score, + measured_value=f"{n_covered}/3 categories", + threshold="all 3 categories (correctness, standards, security)", + evidence=evidence, + remediation=self._create_remediation(missing, primary), + error_message=None, + ) + + # ------------------------------------------------------------------ # + # Per-language tool collection # + # ------------------------------------------------------------------ # + + def _collect_python_tools(self, repository: Repository) -> set[str]: + tools: set[str] = set() + + # pyproject.toml — [tool.X] sections + pyproject = repository.path / "pyproject.toml" + if pyproject.exists(): + try: + with open(pyproject, "rb") as f: + data = tomllib.load(f) + for key, cfg in data.get("tool", {}).items(): + if cfg: # section must have at least one key (not just the header) + tools.add(key.lower()) + except (OSError, tomllib.TOMLDecodeError): + pass + + # setup.cfg — [X] section headers + setup_cfg = repository.path / "setup.cfg" + if setup_cfg.exists(): + try: + parser = configparser.ConfigParser() + parser.read(str(setup_cfg), encoding="utf-8") + for section in parser.sections(): + if parser.options(section): # section must have at least one key + tools.add(section.strip("[]").lower()) + except (OSError, configparser.Error): + pass + + # Standalone config file existence → tool presence + standalone: dict[str, str] = { + "mypy.ini": "mypy", + ".mypy.ini": "mypy", + "ruff.toml": "ruff", + ".ruff.toml": "ruff", + ".flake8": "flake8", + ".pylintrc": "pylint", + "pyrightconfig.json": "pyright", + ".bandit": "bandit", + } + for filename, tool_name in standalone.items(): + if (repository.path / filename).exists(): + tools.add(tool_name) + + # .pre-commit-config.yaml — hook ids + tools |= self._collect_precommit_hooks(repository) + + return tools + + def _collect_go_tools(self, repository: Repository) -> set[str]: + tools: set[str] = set() + + config_names = (".golangci.yml", ".golangci.yaml", ".golangci.toml") + search_dirs = [repository.path] + self._find_go_module_roots(repository) + + for search_dir in search_dirs: + for config_name in config_names: + config_path = search_dir / config_name + if not config_path.exists(): + continue + try: + raw = config_path.read_text(encoding="utf-8") + if config_name.endswith(".toml"): + parsed = tomllib.loads(raw) + else: + parsed = yaml.safe_load(raw) + if not isinstance(parsed, dict): + continue + + linters = parsed.get("linters", {}) + if not isinstance(linters, dict): + continue + # Explicit enable list + enabled = linters.get("enable") or [] + if isinstance(enabled, list): + tools.update(t.lower() for t in enabled if isinstance(t, str)) + + # enable-all implies everything, treat as full coverage + if linters.get("enable-all"): + tools.update(self._GO_TOOLS.keys()) + except (OSError, UnicodeDecodeError, yaml.YAMLError, ValueError): + continue + + return tools + + def _collect_js_tools(self, repository: Repository) -> set[str]: + """Collect JS/TS lint tools from ESLint configs and tsconfig.""" + tools: set[str] = set() + + # JSON-parseable ESLint configs (structural extraction) + eslint_json_configs = [".eslintrc", ".eslintrc.json"] + # YAML ESLint configs — parse structurally with yaml.safe_load + eslint_yaml_configs = [".eslintrc.yaml", ".eslintrc.yml"] + # JS/CJS/MJS configs — text matching only (cannot parse without Node.js) + eslint_js_configs = [ + ".eslintrc.js", + ".eslintrc.cjs", + ".eslintrc.mjs", + "eslint.config.js", + "eslint.config.mjs", + "eslint.config.cjs", + "eslint.config.ts", + ] + + for cfg_name in eslint_json_configs: + cfg_path = repository.path / cfg_name + if not cfg_path.exists(): + continue + try: + raw = cfg_path.read_text(encoding="utf-8") + tools |= self._extract_eslint_tools(raw, is_json=True) + except (OSError, UnicodeDecodeError): + continue + + for cfg_name in eslint_yaml_configs: + cfg_path = repository.path / cfg_name + if not cfg_path.exists(): + continue + try: + raw = cfg_path.read_text(encoding="utf-8") + data = yaml.safe_load(raw) + if isinstance(data, dict): + tools |= self._extract_eslint_tools_from_dict(data) + except (OSError, UnicodeDecodeError, yaml.YAMLError): + continue + + for cfg_name in eslint_js_configs: + cfg_path = repository.path / cfg_name + if not cfg_path.exists(): + continue + try: + raw = cfg_path.read_text(encoding="utf-8") + tools |= self._extract_eslint_tools(raw, is_json=False) + except (OSError, UnicodeDecodeError): + continue + + # package.json — eslintConfig key (structural) only; devDependencies alone + # are insufficient — a plugin must be explicitly enabled in an ESLint config. + pkg_json = repository.path / "package.json" + if pkg_json.exists(): + try: + data = json.loads(pkg_json.read_text(encoding="utf-8")) + if not isinstance(data, dict): + raise json.JSONDecodeError("not a dict", "", 0) + # eslintConfig embedded — parse structurally + eslint_cfg = data.get("eslintConfig", {}) + if isinstance(eslint_cfg, dict) and eslint_cfg: + tools |= self._extract_eslint_tools_from_dict(eslint_cfg) + # prettier key in package.json signals active prettier config + if data.get("prettier"): + tools.add("prettier") + except (OSError, json.JSONDecodeError): + pass + + # Standalone prettier config files + prettier_configs = [ + ".prettierrc", + ".prettierrc.json", + ".prettierrc.yaml", + ".prettierrc.yml", + ".prettierrc.js", + ".prettierrc.cjs", + "prettier.config.js", + "prettier.config.cjs", + "prettier.config.mjs", + ] + for cfg_name in prettier_configs: + if (repository.path / cfg_name).exists(): + tools.add("prettier") + break + + # tsconfig.json strict: true → correctness signal + for tsconfig_path in self._find_tsconfig_files(repository): + try: + raw = tsconfig_path.read_text(encoding="utf-8") + cleaned = self._strip_json_comments(raw) + tsconfig = json.loads(cleaned) + if not isinstance(tsconfig, dict): + continue + compiler = tsconfig.get("compilerOptions", {}) + if not isinstance(compiler, dict): + continue + if compiler.get("strict") or ( + compiler.get("noImplicitAny") and compiler.get("strictNullChecks") + ): + tools.add("@typescript-eslint") # signals correctness tooling + except (OSError, json.JSONDecodeError): + continue + + # .pre-commit-config.yaml + tools |= self._collect_precommit_hooks(repository) + + return tools + + def _find_tsconfig_files(self, repository: Repository) -> list[Path]: + """Find all tsconfig.json files, excluding node_modules/vendor/testdata.""" + found = [] + for tsconfig in repository.path.rglob("tsconfig.json"): + parts = tsconfig.parts + if "node_modules" in parts or "vendor" in parts or "testdata" in parts: + continue + found.append(tsconfig) + return sorted(found) + + @staticmethod + def _strip_json_comments(text: str) -> str: + """Strip // and /* */ comments from JSONC, preserving string contents.""" + out: list[str] = [] + i = 0 + n = len(text) + while i < n: + c = text[i] + if c == '"': + out.append(c) + i += 1 + while i < n and text[i] != '"': + if text[i] == "\\" and i + 1 < n: + out.append(text[i]) + out.append(text[i + 1]) + i += 2 + else: + out.append(text[i]) + i += 1 + if i < n: + out.append(text[i]) + i += 1 + continue + if c == "/" and i + 1 < n and text[i + 1] == "/": + i += 2 + while i < n and text[i] != "\n": + i += 1 + continue + if c == "/" and i + 1 < n and text[i + 1] == "*": + i += 2 + while i + 1 < n and not (text[i] == "*" and text[i + 1] == "/"): + i += 1 + i += 2 + continue + out.append(c) + i += 1 + return "".join(out) + + def _extract_eslint_tools(self, text: str, is_json: bool = False) -> set[str]: + """Extract extends and plugins from ESLint config content. + + For JSON-based configs (.eslintrc.json, eslintConfig in package.json), + parse structurally and walk extends/plugins arrays to avoid false + positives from comments or unrelated text. + + For JS/CJS/MJS configs (not parseable as JSON), fall back to text + matching against the known tool catalog — only alternative without + running Node.js. + """ + tools: set[str] = set() + + if is_json: + try: + data = json.loads(text) + if isinstance(data, dict): + return self._extract_eslint_tools_from_dict(data) + except (json.JSONDecodeError, AttributeError): + pass + # JSON parse failed or yielded non-dict — do not fall back to text + # matching for JSON configs; comments/disabled examples would cause + # false positives. + return tools + + # Text matching fallback for JS/CJS/MJS configs (cannot parse without Node.js) + for candidate in self._JS_TOOLS: + if candidate.lower() in text.lower(): + tools.add(candidate) + return tools + + def _extract_eslint_tools_from_dict(self, data: dict) -> set[str]: + """Structural extraction of ESLint tools from a parsed config dict. + + Inspects extends, plugins, and rule-key namespaces. Works for both + JSON-parsed and YAML-parsed ESLint configs. + """ + tools: set[str] = set() + candidates: list[str] = [] + + extends = data.get("extends", []) + if isinstance(extends, str): + extends = [extends] + if isinstance(extends, list): + candidates.extend(v for v in extends if isinstance(v, str)) + + plugins = data.get("plugins", []) + if isinstance(plugins, str): + plugins = [plugins] + if isinstance(plugins, list): + candidates.extend(v for v in plugins if isinstance(v, str)) + + # Rule-key namespaces (e.g. "@typescript-eslint/no-unused-vars") + rules = data.get("rules", {}) + if isinstance(rules, dict): + for rule_key in rules: + if "/" in rule_key: + candidates.append(rule_key.split("/")[0]) + + joined = " ".join(candidates).lower() + for catalog_entry in self._JS_TOOLS: + if catalog_entry.lower() in joined: + tools.add(catalog_entry) + return tools + + def _collect_precommit_hooks(self, repository: Repository) -> set[str]: + """Parse .pre-commit-config.yaml for hook ids.""" + tools: set[str] = set() + precommit = repository.path / ".pre-commit-config.yaml" + if not precommit.exists(): + return tools + try: + data = yaml.safe_load(precommit.read_text(encoding="utf-8")) + if not isinstance(data, dict): + return tools + repos = data.get("repos", []) + if not isinstance(repos, list): + return tools + for repo_entry in repos: + if not isinstance(repo_entry, dict): + continue + hooks = repo_entry.get("hooks", []) + if not isinstance(hooks, list): + continue + for hook in hooks: + if not isinstance(hook, dict): + continue + hook_id = hook.get("id", "").lower() + if hook_id: + tools.add(hook_id) + except (OSError, yaml.YAMLError): + pass + return tools + + def _collect_ci_tools(self, repository: Repository, language: str) -> set[str]: + """Scan CI workflow files for lint tool invocations in run: steps. + + Parses YAML structurally and extracts only the text of `run:` step + values, avoiding false positives from job names, comments, or + condition strings that happen to contain a tool name. + + Falls back to full-text scan for CI files that cannot be parsed as + YAML (e.g. Travis .travis.yml with complex anchors, or corrupted files). + """ + ci_files: list[Path] = [] + + gh_workflows = repository.path / ".github" / "workflows" + if gh_workflows.exists(): + ci_files.extend(gh_workflows.glob("*.yml")) + ci_files.extend(gh_workflows.glob("*.yaml")) + + for ci_path in ( + repository.path / ".gitlab-ci.yml", + repository.path / ".circleci" / "config.yml", + repository.path / ".travis.yml", + ): + if ci_path.exists(): + ci_files.append(ci_path) + + if language == "Python": + candidates = list(self._PYTHON_TOOLS.keys()) + elif language == "Go": + candidates = list(self._GO_TOOLS.keys()) + else: + candidates = list(self._JS_TOOLS.keys()) + + all_tools: set[str] = set() + + for ci_file in ci_files: + try: + raw = ci_file.read_text(encoding="utf-8") + except (OSError, UnicodeDecodeError): + continue + + run_text = self._extract_ci_run_commands(raw) + run_lower = run_text.lower() + for tool in candidates: + # Use lookarounds instead of \b so @-prefixed names like + # @typescript-eslint also match (@ is non-\w, so \b fails there). + pattern = r"(? str: + """Extract the text of all `run:` step values from a CI YAML file. + + Walks the parsed YAML tree recursively to collect every string value + associated with a `run` key, regardless of nesting depth or CI platform + (GitHub Actions, GitLab CI, CircleCI). Falls back to the full raw text + if the file cannot be parsed. + """ + try: + data = yaml.safe_load(ci_yaml) + except yaml.YAMLError: + return ci_yaml # fallback: full text scan + + if not isinstance(data, dict): + return ci_yaml + + run_chunks: list[str] = [] + + def _walk(node: object) -> None: + if isinstance(node, dict): + for key, value in node.items(): + if key == "run": + if isinstance(value, str): + # GitHub Actions: run: "cmd" + run_chunks.append(value) + elif isinstance(value, dict): + # CircleCI: run: {command: "cmd", ...} + cmd = value.get("command", "") + if isinstance(cmd, str) and cmd: + run_chunks.append(cmd) + _walk(value) + else: + _walk(value) + elif key == "script": + if isinstance(value, list): + # GitLab CI: script: [cmd1, cmd2, ...] + run_chunks.extend(s for s in value if isinstance(s, str)) + else: + _walk(value) + else: + _walk(value) + elif isinstance(node, list): + for item in node: + _walk(item) + + _walk(data) + if not run_chunks: + # YAML parsed successfully but contained no run/script commands — + # return empty rather than the raw file to avoid false positives + # from job names, YAML keys, or other inert text. + return "" + + # Filter lines that are pure installation or echo statements with no + # compound operator (&&, ;, |). These mention a tool without invoking it. + _INERT_PREFIXES = ( + "echo ", + "pip install ", + "pip3 install ", + "npm install ", + "npm ci", + "yarn add ", + "yarn install", + "apt-get install ", + "apt install ", + "brew install ", + "go install ", + ) + filtered: list[str] = [] + for chunk in run_chunks: + lines = [] + for line in chunk.splitlines(): + stripped = line.strip().lower() + has_compound = any(op in stripped for op in ("&&", ";", "|")) + if not has_compound and any( + stripped.startswith(p) for p in _INERT_PREFIXES + ): + continue # skip pure install/echo line + lines.append(line) + if lines: + filtered.append("\n".join(lines)) + + return "\n".join(filtered) + + # ------------------------------------------------------------------ # + # Evidence and remediation helpers # + # ------------------------------------------------------------------ # + + def _build_evidence( + self, + covered: set[str], + tools_by_cat: dict[str, list[str]], + all_tools: set[str], + ) -> list[str]: + evidence: list[str] = [] + for cat in self.CATEGORIES: + if cat in covered: + unique_tools = list(dict.fromkeys(tools_by_cat[cat]))[:3] + evidence.append( + f"Correctness: {', '.join(unique_tools)}" + if cat == "correctness" + else ( + f"Standards: {', '.join(unique_tools)}" + if cat == "standards" + else f"Security: {', '.join(unique_tools)}" + ) + ) + else: + evidence.append(f"Missing category: {cat}") + if all_tools: + evidence.append(f"Tools detected: {', '.join(sorted(all_tools)[:8])}") + return evidence + + def _create_remediation(self, missing: list[str], language: str) -> Remediation: + steps: list[str] = [] + tools: list[str] = [] + commands: list[str] = [] + examples: list[str] = [] + + if language == "Python": + if "correctness" in missing: + steps.append( + "Add mypy or pyright for type/correctness checking: " + "configure [tool.mypy] in pyproject.toml" + ) + tools += ["mypy", "pyright"] + commands += ["pip install mypy", "mypy --strict src/"] + if "standards" in missing: + steps.append( + "Add ruff for style enforcement: configure [tool.ruff] in pyproject.toml" + ) + tools += ["ruff"] + commands += ["pip install ruff", "ruff check ."] + if "security" in missing: + steps.append( + "Add bandit for security analysis: configure [tool.bandit] in pyproject.toml " + "or add bandit pre-commit hook" + ) + tools += ["bandit", "semgrep"] + commands += ["pip install bandit", "bandit -r src/"] + examples.append( + "# pyproject.toml\n" + "[tool.mypy]\nstrict = true\n\n" + '[tool.ruff]\nselect = ["E", "F", "S"] # S = flake8-bandit (security)\n\n' + '[tool.bandit]\ntargets = ["src"]\n' + ) + + elif language == "Go": + if "correctness" in missing: + steps.append( + "Enable errcheck and staticcheck in .golangci.yml linters.enable" + ) + tools += ["golangci-lint"] + commands += ["golangci-lint run --enable errcheck,staticcheck"] + if "standards" in missing: + steps.append("Enable revive or stylecheck in .golangci.yml") + tools += ["golangci-lint"] + if "security" in missing: + steps.append("Enable gosec in .golangci.yml linters.enable") + tools += ["gosec"] + commands += ["golangci-lint run --enable gosec"] + examples.append( + "# .golangci.yml\nlinters:\n enable:\n" + " - errcheck\n - staticcheck\n" + " - revive\n - stylecheck\n - gosec\n" + ) + + else: # JS/TS + if "correctness" in missing: + steps.append( + "Add @typescript-eslint/recommended-type-checked to ESLint extends" + ) + tools += ["@typescript-eslint/eslint-plugin"] + commands += ["npm install --save-dev @typescript-eslint/eslint-plugin"] + if "standards" in missing: + steps.append( + "Add eslint:recommended or airbnb config to ESLint extends" + ) + tools += ["eslint"] + commands += ["npm install --save-dev eslint"] + if "security" in missing: + steps.append("Add eslint-plugin-security to ESLint plugins") + tools += ["eslint-plugin-security"] + commands += ["npm install --save-dev eslint-plugin-security"] + examples.append( + '// .eslintrc.json\n{\n "extends": [\n' + ' "eslint:recommended",\n' + ' "plugin:@typescript-eslint/recommended-type-checked",\n' + ' "plugin:security/recommended"\n' + " ]\n}\n" + ) + + return Remediation( + summary=f"Extend lint config to cover missing categories: {', '.join(missing)}", + steps=steps, + tools=tools, + commands=commands, + examples=examples, + citations=[ + Citation( + source="agentready", + title="Lint Config Coverage — Issue #511", + url="https://github.com/ambient-code/agentready/issues/511", + relevance="Feature rationale and tool categorization", + ) + ], + ) diff --git a/src/agentready/data/default-weights.yaml b/src/agentready/data/default-weights.yaml index 44098e95..d5df4420 100644 --- a/src/agentready/data/default-weights.yaml +++ b/src/agentready/data/default-weights.yaml @@ -1,6 +1,6 @@ # Default Tier-Based Weight Distribution # -# This file defines the default weights for all 28 attributes. +# This file defines the default weights for all 29 attributes. # Weights are based on evidence from ETH Zurich (Feb 2026), Anthropic, # Red Hat best practices (April 2026), and Cursor agent guidelines. # @@ -8,7 +8,7 @@ # agent effectiveness. Documentation helps only marginally (+4%). # # Tier 1 (Essential): 58% total (9 attributes) -# Tier 2 (Critical): 27% total (3% each, 9 attributes) +# Tier 2 (Critical): 27% total (10 attributes, mixed weights) # Tier 3 (Important): 13% total (varies, 8 attributes) # Tier 4 (Advanced): 2% total (1% each, 2 attributes) # TOTAL: 100% (sum to 1.0) @@ -35,6 +35,11 @@ # - Added adr_frontmatter_completeness (T3, 2%): ADR structured-metadata coverage # - Reduced architecture_decisions (2% -> 1%) to fund new weight # - Reduced progressive_disclosure (2% -> 1%) to fund new weight +# +# Changes in v2.4.0 (feat/lint-config-coverage, issue #511): +# - Added lint_config_coverage (T2, 2%): lint depth across correctness/standards/security +# - Reduced gitignore_completeness (3% -> 2%) to partially fund new weight +# - Reduced pattern_references (3% -> 2%) to partially fund new weight # Tier 1 (Essential) - 58% total weight test_execution: 0.11 # 5.1 - Test Execution & Coverage @@ -47,16 +52,17 @@ standard_layout: 0.05 # 4.1 - Standard Project Layouts lock_files: 0.05 # 6.1 - Dependency Pinning for Reproducibility dependency_security: 0.05 # 6.2 - Dependency Security & Vulnerability Scanning -# Tier 2 (Critical) - 27% total weight (3% each) +# Tier 2 (Critical) - 27% total weight deterministic_enforcement: 0.03 # 5.3 - Hooks & Lint Rules (deterministic > advisory) conventional_commits: 0.03 # 7.1 - Conventional Commit Messages -gitignore_completeness: 0.03 # 7.2 - .gitignore Completeness +gitignore_completeness: 0.02 # 7.2 - .gitignore Completeness (reduced 3%→2%) one_command_setup: 0.03 # 8.1 - One-Command Build/Setup inline_documentation: 0.03 # 2.2 - Inline Documentation file_size_limits: 0.03 # 1.3 - File Size Limits separation_of_concerns: 0.03 # 4.2 - Separation of Concerns -pattern_references: 0.03 # 17.1 - Pattern References for Common Changes +pattern_references: 0.02 # 17.1 - Pattern References (reduced 3%→2%) design_intent: 0.03 # 17.2 - Design Intent Documentation (moved from T3) +lint_config_coverage: 0.02 # 5.4 - Lint Config Coverage (issue #511, new) # Tier 3 (Important) - 13% total weight (8 attributes) architecture_decisions: 0.01 # 2.3 - Architecture Decision Records diff --git a/tests/unit/test_assessors_code_quality.py b/tests/unit/test_assessors_code_quality.py index 4629dd03..407950ea 100644 --- a/tests/unit/test_assessors_code_quality.py +++ b/tests/unit/test_assessors_code_quality.py @@ -6,6 +6,7 @@ from agentready.assessors.code_quality import ( CyclomaticComplexityAssessor, + LintConfigCoverageAssessor, TypeAnnotationsAssessor, ) from agentready.models.repository import Repository @@ -422,3 +423,418 @@ def test_radon_exception_returns_error(self, tmp_path): assert finding.status == "error" assert "radon crashed" in finding.error_message + + +# ============================================================================= +# LintConfigCoverageAssessor +# ============================================================================= + + +def _make_repo(tmp_path, languages=None): + (tmp_path / ".git").mkdir() + return Repository( + path=tmp_path, + name="test-repo", + url=None, + branch="main", + commit_hash="abc123", + languages=languages or {"Python": 100}, + total_files=5, + total_lines=200, + ) + + +class TestLintConfigCoverageAssessorPython: + """Tests for Python lint config coverage detection.""" + + def test_all_three_categories_passes(self, tmp_path): + """Correctness (mypy) + standards (ruff) + security (bandit) → pass.""" + repo = _make_repo(tmp_path) + pyproject = tmp_path / "pyproject.toml" + pyproject.write_text( + "[tool.mypy]\nstrict = true\n\n" + "[tool.ruff]\nselect = ['E']\n\n" + "[tool.bandit]\ntargets = ['src']\n" + ) + finding = LintConfigCoverageAssessor().assess(repo) + + assert finding.status == "pass" + assert finding.score == 100.0 + assert "3/3" in finding.measured_value + + def test_standards_only_fails_with_partial_credit(self, tmp_path): + """Only black (standards) → 1/3 categories, score=33, fail. + + Note: ruff covers correctness+standards; use black (standards only) here. + """ + repo = _make_repo(tmp_path) + pyproject = tmp_path / "pyproject.toml" + pyproject.write_text("[tool.black]\nline-length = 88\n") + finding = LintConfigCoverageAssessor().assess(repo) + + assert finding.status == "fail" + assert finding.score < 50 + assert "1/3" in finding.measured_value + + def test_correctness_and_standards_partial(self, tmp_path): + """mypy + ruff → 2/3 → score≈67, fail (below pass threshold of 100).""" + repo = _make_repo(tmp_path) + (tmp_path / "mypy.ini").write_text("[mypy]\nstrict = True\n") + (tmp_path / "ruff.toml").write_text("[lint]\n") + finding = LintConfigCoverageAssessor().assess(repo) + + assert finding.status == "fail" + assert finding.score >= 60.0 + assert "2/3" in finding.measured_value + + def test_precommit_bandit_adds_security(self, tmp_path): + """bandit in pre-commit hooks counts as security category.""" + repo = _make_repo(tmp_path) + # standards + correctness via pyproject, security via pre-commit + (tmp_path / "pyproject.toml").write_text( + "[tool.mypy]\nstrict = true\n\n[tool.ruff]\nselect=['E']\n" + ) + precommit = tmp_path / ".pre-commit-config.yaml" + precommit.write_text( + "repos:\n" + " - repo: https://github.com/pycqa/bandit\n" + " rev: 1.7.5\n" + " hooks:\n" + " - id: bandit\n" + ) + finding = LintConfigCoverageAssessor().assess(repo) + + assert finding.status == "pass" + assert finding.score == 100.0 + + def test_ci_workflow_bandit_adds_security(self, tmp_path): + """bandit mention in CI workflow counts as security coverage.""" + repo = _make_repo(tmp_path) + (tmp_path / "pyproject.toml").write_text( + "[tool.mypy]\nstrict = true\n\n[tool.ruff]\nselect=['E']\n" + ) + wf_dir = tmp_path / ".github" / "workflows" + wf_dir.mkdir(parents=True) + (wf_dir / "ci.yml").write_text( + "name: CI\non: [push]\njobs:\n lint:\n steps:\n" + " - run: bandit -r src/\n" + ) + finding = LintConfigCoverageAssessor().assess(repo) + + assert finding.status == "pass" + + def test_no_lint_config_fails_with_zero(self, tmp_path): + """Repo with no lint configuration → 0/3, score=0, fail.""" + repo = _make_repo(tmp_path) + finding = LintConfigCoverageAssessor().assess(repo) + + assert finding.status == "fail" + assert finding.score == 0.0 + + def test_remediation_lists_missing_categories(self, tmp_path): + """Remediation steps mention the missing category (security).""" + repo = _make_repo(tmp_path) + # ruff covers correctness+standards; only security missing + (tmp_path / "ruff.toml").write_text("[lint]\n") + (tmp_path / "mypy.ini").write_text("[mypy]\nstrict = True\n") + finding = LintConfigCoverageAssessor().assess(repo) + + assert finding.remediation is not None + combined = " ".join(finding.remediation.steps).lower() + assert "security" in combined or "bandit" in combined + + def test_not_applicable_for_unsupported_language(self, tmp_path): + """Repo with no supported language returns not_applicable.""" + repo = _make_repo(tmp_path, languages={"Haskell": 100}) + finding = LintConfigCoverageAssessor().assess(repo) + + assert finding.status == "not_applicable" + + def test_empty_pyproject_tool_section_not_counted(self, tmp_path): + """Empty [tool.ruff] section in pyproject.toml must not award category credit. + + A bare section header with no keys is indistinguishable from a mis-pasted + config; only a section with at least one explicit key is counted. + """ + repo = _make_repo(tmp_path) + # [tool.ruff] has no keys — ruff is not meaningfully configured + (tmp_path / "pyproject.toml").write_text("[tool.ruff]\n") + finding = LintConfigCoverageAssessor().assess(repo) + + assert finding.status == "fail" + assert finding.score == 0.0 + assert "0/3" in finding.measured_value + + def test_empty_setup_cfg_section_not_counted(self, tmp_path): + """Empty [ruff] section in setup.cfg must not award category credit. + + Mirrors test_empty_pyproject_tool_section_not_counted for setup.cfg. + """ + repo = _make_repo(tmp_path) + (tmp_path / "setup.cfg").write_text("[ruff]\n") + finding = LintConfigCoverageAssessor().assess(repo) + + assert finding.status == "fail" + assert finding.score == 0.0 + assert "0/3" in finding.measured_value + + +class TestLintConfigCoverageAssessorGo: + """Tests for Go lint config coverage detection.""" + + def _make_go_repo(self, tmp_path): + (tmp_path / ".git").mkdir(exist_ok=True) + (tmp_path / "go.mod").write_text("module example.com/myapp\n\ngo 1.21\n") + return Repository( + path=tmp_path, + name="test-go-repo", + url=None, + branch="main", + commit_hash="abc123", + languages={"Go": 100}, + total_files=5, + total_lines=200, + ) + + def test_golangci_all_three_categories(self, tmp_path): + """golangci config with errcheck + revive + gosec → pass.""" + repo = self._make_go_repo(tmp_path) + (tmp_path / ".golangci.yml").write_text( + "linters:\n" + " enable:\n" + " - errcheck\n" + " - staticcheck\n" + " - revive\n" + " - gosec\n" + ) + finding = LintConfigCoverageAssessor().assess(repo) + + assert finding.status == "pass" + assert finding.score == 100.0 + + def test_golangci_correctness_only_partial(self, tmp_path): + """golangci with only errcheck → 1/3, fail.""" + repo = self._make_go_repo(tmp_path) + (tmp_path / ".golangci.yml").write_text("linters:\n enable:\n - errcheck\n") + finding = LintConfigCoverageAssessor().assess(repo) + + assert finding.status == "fail" + assert finding.score < 50 + + def test_golangci_enable_all(self, tmp_path): + """golangci enable-all covers all categories.""" + repo = self._make_go_repo(tmp_path) + (tmp_path / ".golangci.yml").write_text("linters:\n enable-all: true\n") + finding = LintConfigCoverageAssessor().assess(repo) + + assert finding.status == "pass" + + +class TestLintConfigCoverageAssessorJS: + """Tests for JavaScript/TypeScript lint config coverage detection.""" + + def _make_js_repo(self, tmp_path, lang="JavaScript"): + (tmp_path / ".git").mkdir(exist_ok=True) + (tmp_path / "package.json").write_text('{"name":"test","version":"1.0.0"}\n') + return Repository( + path=tmp_path, + name="test-js-repo", + url=None, + branch="main", + commit_hash="abc123", + languages={lang: 100}, + total_files=5, + total_lines=200, + ) + + def test_js_only_full_pass_without_typescript(self, tmp_path): + """Pure JS repo: eslint:recommended (correctness+standards) + security → 3/3 pass.""" + repo = self._make_js_repo(tmp_path, "JavaScript") + (tmp_path / ".eslintrc.json").write_text( + json.dumps( + { + "extends": ["eslint:recommended", "plugin:security/recommended"], + } + ) + ) + finding = LintConfigCoverageAssessor().assess(repo) + + assert finding.status == "pass" + assert finding.score == 100.0 + assert "3/3" in finding.measured_value + + def test_eslint_with_ts_and_security(self, tmp_path): + """ESLint with @typescript-eslint + airbnb + security plugin → pass.""" + repo = self._make_js_repo(tmp_path, "TypeScript") + (tmp_path / ".eslintrc.json").write_text( + json.dumps( + { + "extends": [ + "airbnb", + "plugin:@typescript-eslint/recommended-type-checked", + "plugin:security/recommended", + ] + } + ) + ) + finding = LintConfigCoverageAssessor().assess(repo) + + assert finding.status == "pass" + + def test_tsconfig_strict_counts_for_correctness(self, tmp_path): + """tsconfig strict:true + eslint:recommended → correctness + standards.""" + repo = self._make_js_repo(tmp_path, "TypeScript") + (tmp_path / "tsconfig.json").write_text('{"compilerOptions": {"strict": true}}') + (tmp_path / ".eslintrc.json").write_text('{"extends": ["eslint:recommended"]}') + finding = LintConfigCoverageAssessor().assess(repo) + + assert finding.score >= 60 # at least 2/3 + assert "2/3" in finding.measured_value or "3/3" in finding.measured_value + + def test_installed_but_unused_plugin_gets_no_credit(self, tmp_path): + """eslint-plugin-security in devDependencies but NOT in ESLint config gets no credit. + + Installing a plugin does not activate it — it must appear in extends/plugins. + """ + repo = self._make_js_repo(tmp_path) + (tmp_path / "package.json").write_text( + json.dumps( + { + "devDependencies": { + "eslint": "^8", + "eslint-plugin-security": "^1", + "@typescript-eslint/eslint-plugin": "^5", + } + } + ) + ) + # Only eslint:recommended in extends — security plugin installed but not enabled + (tmp_path / ".eslintrc.json").write_text('{"extends": ["eslint:recommended"]}') + finding = LintConfigCoverageAssessor().assess(repo) + + # correctness+standards from eslint:recommended; security devDep alone ≠ active + assert finding.score < 100 + assert "2/3" in finding.measured_value + assert "Missing category: security" in finding.evidence + + def test_eslint_json_comment_not_false_positive(self, tmp_path): + """Tool name in a JSON comment must not count as coverage (JSON parsed structurally).""" + repo = self._make_js_repo(tmp_path, "TypeScript") + # Only eslint:recommended in extends — security mentioned only in a comment + (tmp_path / ".eslintrc.json").write_text( + '{"extends": ["eslint:recommended"], "_comment": "we chose not to use plugin:security"}' + ) + finding = LintConfigCoverageAssessor().assess(repo) + + # Should detect standards (eslint:recommended) but NOT security from the comment + assert "Missing category: security" in finding.evidence + + def test_prettier_standalone_config_counts(self, tmp_path): + """A standalone .prettierrc file signals active prettier (standards).""" + repo = self._make_js_repo(tmp_path) + (tmp_path / ".prettierrc.json").write_text('{"singleQuote": true}') + # Also add correctness and security via ESLint to get a meaningful result + (tmp_path / ".eslintrc.json").write_text( + '{"extends": ["plugin:@typescript-eslint/recommended", "plugin:security/recommended"]}' + ) + finding = LintConfigCoverageAssessor().assess(repo) + + assert finding.status == "pass" + assert finding.score == 100.0 + + +class TestLintConfigCoverageAssessorCI: + """Tests for CI workflow step-aware scanning.""" + + def _make_python_repo_ci(self, tmp_path): + (tmp_path / ".git").mkdir() + return Repository( + path=tmp_path, + name="test-ci-repo", + url=None, + branch="main", + commit_hash="abc123", + languages={"Python": 100}, + total_files=5, + total_lines=200, + ) + + def test_ci_tool_in_run_step_detected(self, tmp_path): + """Tool name inside a run: step is detected as coverage.""" + repo = self._make_python_repo_ci(tmp_path) + (tmp_path / "pyproject.toml").write_text( + "[tool.mypy]\nstrict = true\n\n[tool.ruff]\nselect=['E']\n" + ) + wf_dir = tmp_path / ".github" / "workflows" + wf_dir.mkdir(parents=True) + (wf_dir / "ci.yml").write_text( + "name: CI\non: [push]\njobs:\n" + " security:\n runs-on: ubuntu-latest\n steps:\n" + " - run: pip install bandit && bandit -r src/\n" + ) + finding = LintConfigCoverageAssessor().assess(repo) + + assert finding.status == "pass" + assert finding.score == 100.0 + + def test_ci_tool_in_comment_not_detected(self, tmp_path): + """Tool name appearing only in a YAML comment must not count as coverage.""" + repo = self._make_python_repo_ci(tmp_path) + (tmp_path / "pyproject.toml").write_text( + "[tool.mypy]\nstrict = true\n\n[tool.ruff]\nselect=['E']\n" + ) + wf_dir = tmp_path / ".github" / "workflows" + wf_dir.mkdir(parents=True) + # bandit only in a comment, not in any run: step + (wf_dir / "ci.yml").write_text( + "name: CI\non: [push]\n" + "# we considered bandit but skipped it\n" + "jobs:\n lint:\n runs-on: ubuntu-latest\n steps:\n" + " - run: ruff check .\n" + ) + finding = LintConfigCoverageAssessor().assess(repo) + + assert "Missing category: security" in finding.evidence + + def test_gitlab_ci_script_field_detected(self, tmp_path): + """GitLab CI script: list commands are scanned for tool invocations.""" + repo = self._make_python_repo_ci(tmp_path) + (tmp_path / "pyproject.toml").write_text( + "[tool.mypy]\nstrict = true\n\n[tool.ruff]\nselect=['E']\n" + ) + # GitLab CI uses script: list, not run: + (tmp_path / ".gitlab-ci.yml").write_text( + "stages:\n - lint\n\n" + "security-scan:\n" + " stage: lint\n" + " script:\n" + " - pip install bandit\n" + " - bandit -r src/\n" + ) + finding = LintConfigCoverageAssessor().assess(repo) + + assert finding.status == "pass" + assert finding.score == 100.0 + + def test_circleci_run_command_field_detected(self, tmp_path): + """CircleCI run.command dict field is scanned for tool invocations.""" + repo = self._make_python_repo_ci(tmp_path) + (tmp_path / "pyproject.toml").write_text( + "[tool.mypy]\nstrict = true\n\n[tool.ruff]\nselect=['E']\n" + ) + circleci_dir = tmp_path / ".circleci" + circleci_dir.mkdir() + # CircleCI stores commands in run: {command: "..."} dicts + (circleci_dir / "config.yml").write_text( + "version: 2.1\n" + "jobs:\n" + " security:\n" + " steps:\n" + " - run:\n" + " name: Run bandit security check\n" + " command: bandit -r src/\n" + ) + finding = LintConfigCoverageAssessor().assess(repo) + + assert finding.status == "pass" + assert finding.score == 100.0 diff --git a/uv.lock b/uv.lock index 43c5067c..d0dd6cf3 100644 --- a/uv.lock +++ b/uv.lock @@ -4,7 +4,7 @@ requires-python = ">=3.12" [[package]] name = "agentready" -version = "2.47.0" +version = "2.49.0" source = { editable = "." } dependencies = [ { name = "anthropic" },