From d0de45dfa2b9535464544a1720c102a9272b801b Mon Sep 17 00:00:00 2001 From: andyluo7 Date: Wed, 5 Aug 2026 14:08:21 -0700 Subject: [PATCH 1/2] Add federated AMD Quark skill family --- .claude-plugin/marketplace.json | 21 +- .codex-plugin/plugin.json | 23 +- .cursor-plugin/marketplace.json | 23 +- .github/scripts/check.sh | 1 + .github/scripts/import_external_skills.py | 615 ++++++-- .github/scripts/sources.yml | 67 +- .github/scripts/validate_quark_federation.py | 185 +++ README.md | 26 + docs/federating-your-repo.md | 51 + plugin-metadata.json | 2 + skills/quark-env-preflight/.federated.json | 10 + skills/quark-env-preflight/SKILL.md | 113 ++ .../references/quark-torch-quant-plan.md | 209 +++ skills/quark-env-preflight/skill-card.md | 13 + skills/quark-install/.federated.json | 10 + skills/quark-install/SKILL.md | 165 +++ skills/quark-install/skill-card.md | 13 + .../quark-onnx-autosearch-pro/.federated.json | 10 + skills/quark-onnx-autosearch-pro/SKILL.md | 304 ++++ .../presets-reference.md | 72 + .../quark-onnx-autosearch-pro/skill-card.md | 13 + skills/quark-onnx-debug/.federated.json | 10 + skills/quark-onnx-debug/SKILL.md | 280 ++++ skills/quark-onnx-debug/skill-card.md | 13 + skills/quark-onnx-install/.federated.json | 10 + skills/quark-onnx-install/SKILL.md | 277 ++++ skills/quark-onnx-install/skill-card.md | 13 + .../quark-onnx-model-intake/.federated.json | 10 + skills/quark-onnx-model-intake/SKILL.md | 356 +++++ .../references/quark-workspace-validate.md | 113 ++ skills/quark-onnx-model-intake/skill-card.md | 13 + skills/quark-onnx-ptq/.federated.json | 10 + skills/quark-onnx-ptq/SKILL.md | 344 +++++ .../quark-onnx-ptq/example-xint8-yolov8n.md | 154 ++ .../references/quark-onnx-quant-plan.md | 348 +++++ skills/quark-onnx-ptq/skill-card.md | 13 + .../.federated.json | 10 + skills/quark-onnx-result-validator/SKILL.md | 216 +++ .../quant_validation_onnx.py | 1134 +++++++++++++++ .../run_validation.py | 138 ++ .../quark-onnx-result-validator/skill-card.md | 13 + skills/quark-torch-debug/.federated.json | 10 + skills/quark-torch-debug/SKILL.md | 172 +++ skills/quark-torch-debug/skill-card.md | 13 + skills/quark-torch-export/.federated.json | 10 + skills/quark-torch-export/SKILL.md | 167 +++ .../references/quark-torch-quant-plan.md | 209 +++ skills/quark-torch-export/skill-card.md | 13 + .../.federated.json | 10 + .../SKILL.md | 209 +++ .../skill-card.md | 13 + skills/quark-torch-install/.federated.json | 10 + skills/quark-torch-install/SKILL.md | 121 ++ skills/quark-torch-install/skill-card.md | 13 + skills/quark-torch-llm-eval/.federated.json | 10 + skills/quark-torch-llm-eval/SKILL.md | 351 +++++ skills/quark-torch-llm-eval/backends.md | 303 ++++ .../quark-torch-llm-eval/eval-frameworks.md | 453 ++++++ .../quark-torch-llm-eval/model-inspection.md | 318 +++++ skills/quark-torch-llm-eval/skill-card.md | 13 + skills/quark-torch-llm-eval/templates.md | 118 ++ .../quark-torch-llm-eval/troubleshooting.md | 332 +++++ .../quark-torch-llm-ptq-eval/.federated.json | 10 + skills/quark-torch-llm-ptq-eval/SKILL.md | 180 +++ .../example-fp8-qwen3-8b.md | 81 ++ skills/quark-torch-llm-ptq-eval/skill-card.md | 13 + .../quark-torch-model-intake/.federated.json | 10 + skills/quark-torch-model-intake/SKILL.md | 192 +++ .../references/quark-torch-quant-plan.md | 209 +++ .../references/quark-workspace-validate.md | 113 ++ skills/quark-torch-model-intake/skill-card.md | 13 + skills/quark-torch-ptq/.federated.json | 10 + skills/quark-torch-ptq/SKILL.md | 259 ++++ .../quark-torch-ptq/example-fp8-qwen3-8b.md | 64 + .../references/quark-torch-quant-plan.md | 209 +++ skills/quark-torch-ptq/skill-card.md | 13 + .../.federated.json | 10 + skills/quark-torch-result-validator/SKILL.md | 170 +++ .../quant_validation.py | 1238 +++++++++++++++++ .../run_validation.py | 121 ++ .../skill-card.md | 13 + 81 files changed, 11130 insertions(+), 87 deletions(-) mode change 100644 => 100755 .github/scripts/import_external_skills.py create mode 100755 .github/scripts/validate_quark_federation.py create mode 100644 skills/quark-env-preflight/.federated.json create mode 100644 skills/quark-env-preflight/SKILL.md create mode 100644 skills/quark-env-preflight/references/quark-torch-quant-plan.md create mode 100644 skills/quark-env-preflight/skill-card.md create mode 100644 skills/quark-install/.federated.json create mode 100644 skills/quark-install/SKILL.md create mode 100644 skills/quark-install/skill-card.md create mode 100644 skills/quark-onnx-autosearch-pro/.federated.json create mode 100644 skills/quark-onnx-autosearch-pro/SKILL.md create mode 100644 skills/quark-onnx-autosearch-pro/presets-reference.md create mode 100644 skills/quark-onnx-autosearch-pro/skill-card.md create mode 100644 skills/quark-onnx-debug/.federated.json create mode 100644 skills/quark-onnx-debug/SKILL.md create mode 100644 skills/quark-onnx-debug/skill-card.md create mode 100644 skills/quark-onnx-install/.federated.json create mode 100644 skills/quark-onnx-install/SKILL.md create mode 100644 skills/quark-onnx-install/skill-card.md create mode 100644 skills/quark-onnx-model-intake/.federated.json create mode 100644 skills/quark-onnx-model-intake/SKILL.md create mode 100644 skills/quark-onnx-model-intake/references/quark-workspace-validate.md create mode 100644 skills/quark-onnx-model-intake/skill-card.md create mode 100644 skills/quark-onnx-ptq/.federated.json create mode 100644 skills/quark-onnx-ptq/SKILL.md create mode 100644 skills/quark-onnx-ptq/example-xint8-yolov8n.md create mode 100644 skills/quark-onnx-ptq/references/quark-onnx-quant-plan.md create mode 100644 skills/quark-onnx-ptq/skill-card.md create mode 100644 skills/quark-onnx-result-validator/.federated.json create mode 100644 skills/quark-onnx-result-validator/SKILL.md create mode 100644 skills/quark-onnx-result-validator/quant_validation_onnx.py create mode 100644 skills/quark-onnx-result-validator/run_validation.py create mode 100644 skills/quark-onnx-result-validator/skill-card.md create mode 100644 skills/quark-torch-debug/.federated.json create mode 100644 skills/quark-torch-debug/SKILL.md create mode 100644 skills/quark-torch-debug/skill-card.md create mode 100644 skills/quark-torch-export/.federated.json create mode 100644 skills/quark-torch-export/SKILL.md create mode 100644 skills/quark-torch-export/references/quark-torch-quant-plan.md create mode 100644 skills/quark-torch-export/skill-card.md create mode 100644 skills/quark-torch-file2file-quantization/.federated.json create mode 100644 skills/quark-torch-file2file-quantization/SKILL.md create mode 100644 skills/quark-torch-file2file-quantization/skill-card.md create mode 100644 skills/quark-torch-install/.federated.json create mode 100644 skills/quark-torch-install/SKILL.md create mode 100644 skills/quark-torch-install/skill-card.md create mode 100644 skills/quark-torch-llm-eval/.federated.json create mode 100644 skills/quark-torch-llm-eval/SKILL.md create mode 100644 skills/quark-torch-llm-eval/backends.md create mode 100644 skills/quark-torch-llm-eval/eval-frameworks.md create mode 100644 skills/quark-torch-llm-eval/model-inspection.md create mode 100644 skills/quark-torch-llm-eval/skill-card.md create mode 100644 skills/quark-torch-llm-eval/templates.md create mode 100644 skills/quark-torch-llm-eval/troubleshooting.md create mode 100644 skills/quark-torch-llm-ptq-eval/.federated.json create mode 100644 skills/quark-torch-llm-ptq-eval/SKILL.md create mode 100644 skills/quark-torch-llm-ptq-eval/example-fp8-qwen3-8b.md create mode 100644 skills/quark-torch-llm-ptq-eval/skill-card.md create mode 100644 skills/quark-torch-model-intake/.federated.json create mode 100644 skills/quark-torch-model-intake/SKILL.md create mode 100644 skills/quark-torch-model-intake/references/quark-torch-quant-plan.md create mode 100644 skills/quark-torch-model-intake/references/quark-workspace-validate.md create mode 100644 skills/quark-torch-model-intake/skill-card.md create mode 100644 skills/quark-torch-ptq/.federated.json create mode 100644 skills/quark-torch-ptq/SKILL.md create mode 100644 skills/quark-torch-ptq/example-fp8-qwen3-8b.md create mode 100644 skills/quark-torch-ptq/references/quark-torch-quant-plan.md create mode 100644 skills/quark-torch-ptq/skill-card.md create mode 100644 skills/quark-torch-result-validator/.federated.json create mode 100644 skills/quark-torch-result-validator/SKILL.md create mode 100644 skills/quark-torch-result-validator/quant_validation.py create mode 100644 skills/quark-torch-result-validator/run_validation.py create mode 100644 skills/quark-torch-result-validator/skill-card.md diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index de99b2f..52d06e4 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -17,9 +17,26 @@ "./skills/local-ai-use", "./skills/local-ai-app-integration", "./skills/serving-llms-on-instinct", - "./skills/tracelens-analysis-orchestrator" + "./skills/tracelens-analysis-orchestrator", + "./skills/quark-env-preflight", + "./skills/quark-install", + "./skills/quark-onnx-autosearch-pro", + "./skills/quark-onnx-debug", + "./skills/quark-onnx-install", + "./skills/quark-onnx-model-intake", + "./skills/quark-onnx-ptq", + "./skills/quark-onnx-result-validator", + "./skills/quark-torch-debug", + "./skills/quark-torch-export", + "./skills/quark-torch-file2file-quantization", + "./skills/quark-torch-install", + "./skills/quark-torch-llm-eval", + "./skills/quark-torch-llm-ptq-eval", + "./skills/quark-torch-model-intake", + "./skills/quark-torch-ptq", + "./skills/quark-torch-result-validator" ], - "description": "AMD's verified Agent Skills in one plugin: route image/audio through local AI on Ryzen AI, serve LLMs on AMD Instinct GPUs with vLLM, and analyze GPU kernel and PyTorch trace performance." + "description": "AMD's verified Agent Skills in one plugin: quantize and validate PyTorch or ONNX models with AMD Quark, route image/audio through local AI on Ryzen AI, serve LLMs on AMD Instinct GPUs with vLLM, and analyze GPU kernel and PyTorch trace performance." } ] } diff --git a/.codex-plugin/plugin.json b/.codex-plugin/plugin.json index cd38ee1..0b77970 100644 --- a/.codex-plugin/plugin.json +++ b/.codex-plugin/plugin.json @@ -12,6 +12,8 @@ "amd", "rocm", "hip", + "quark", + "quantization", "ryzen-ai", "vllm", "lemonade", @@ -21,12 +23,29 @@ "./skills/local-ai-use", "./skills/local-ai-app-integration", "./skills/serving-llms-on-instinct", - "./skills/tracelens-analysis-orchestrator" + "./skills/tracelens-analysis-orchestrator", + "./skills/quark-env-preflight", + "./skills/quark-install", + "./skills/quark-onnx-autosearch-pro", + "./skills/quark-onnx-debug", + "./skills/quark-onnx-install", + "./skills/quark-onnx-model-intake", + "./skills/quark-onnx-ptq", + "./skills/quark-onnx-result-validator", + "./skills/quark-torch-debug", + "./skills/quark-torch-export", + "./skills/quark-torch-file2file-quantization", + "./skills/quark-torch-install", + "./skills/quark-torch-llm-eval", + "./skills/quark-torch-llm-ptq-eval", + "./skills/quark-torch-model-intake", + "./skills/quark-torch-ptq", + "./skills/quark-torch-result-validator" ], "interface": { "displayName": "AMD Skills", "shortDescription": "Agent Skills for AMD-optimized workflows.", - "longDescription": "AMD's verified Agent Skills in one plugin: route image/audio through local AI on Ryzen AI, serve LLMs on AMD Instinct GPUs with vLLM, and analyze GPU kernel and PyTorch trace performance.", + "longDescription": "AMD's verified Agent Skills in one plugin: quantize and validate PyTorch or ONNX models with AMD Quark, route image/audio through local AI on Ryzen AI, serve LLMs on AMD Instinct GPUs with vLLM, and analyze GPU kernel and PyTorch trace performance.", "developerName": "AMD", "category": "Developer Tools", "capabilities": [ diff --git a/.cursor-plugin/marketplace.json b/.cursor-plugin/marketplace.json index 62c7697..9a20c60 100644 --- a/.cursor-plugin/marketplace.json +++ b/.cursor-plugin/marketplace.json @@ -16,9 +16,26 @@ "./skills/local-ai-use", "./skills/local-ai-app-integration", "./skills/serving-llms-on-instinct", - "./skills/tracelens-analysis-orchestrator" + "./skills/tracelens-analysis-orchestrator", + "./skills/quark-env-preflight", + "./skills/quark-install", + "./skills/quark-onnx-autosearch-pro", + "./skills/quark-onnx-debug", + "./skills/quark-onnx-install", + "./skills/quark-onnx-model-intake", + "./skills/quark-onnx-ptq", + "./skills/quark-onnx-result-validator", + "./skills/quark-torch-debug", + "./skills/quark-torch-export", + "./skills/quark-torch-file2file-quantization", + "./skills/quark-torch-install", + "./skills/quark-torch-llm-eval", + "./skills/quark-torch-llm-ptq-eval", + "./skills/quark-torch-model-intake", + "./skills/quark-torch-ptq", + "./skills/quark-torch-result-validator" ], - "description": "AMD's verified Agent Skills in one plugin: route image/audio through local AI on Ryzen AI, serve LLMs on AMD Instinct GPUs with vLLM, and analyze GPU kernel and PyTorch trace performance.", + "description": "AMD's verified Agent Skills in one plugin: quantize and validate PyTorch or ONNX models with AMD Quark, route image/audio through local AI on Ryzen AI, serve LLMs on AMD Instinct GPUs with vLLM, and analyze GPU kernel and PyTorch trace performance.", "author": { "name": "AMD" }, @@ -29,6 +46,8 @@ "amd", "rocm", "hip", + "quark", + "quantization", "ryzen-ai", "vllm", "lemonade", diff --git a/.github/scripts/check.sh b/.github/scripts/check.sh index bde1a03..9e62948 100755 --- a/.github/scripts/check.sh +++ b/.github/scripts/check.sh @@ -19,6 +19,7 @@ usage() { case "${1:-}" in "") uv run .github/scripts/validate_skills.py + uv run .github/scripts/validate_quark_federation.py uv run .github/scripts/generate_cursor_marketplace.py --check uv run .github/scripts/generate_codex_plugin.py --check ;; diff --git a/.github/scripts/import_external_skills.py b/.github/scripts/import_external_skills.py old mode 100644 new mode 100755 index fe420be..41e4250 --- a/.github/scripts/import_external_skills.py +++ b/.github/scripts/import_external_skills.py @@ -12,6 +12,11 @@ 2. Copies each named skill folder into `skills//`. When refreshing an existing import, a local `evals/` subdirectory is kept if the upstream skill folder does not ship one (catalog-authored behavioral tests). + Sources that set `resolve_wrappers: true` may expose lightweight public + wrappers whose body points at a repository-owned implementation SKILL.md. + The importer resolves that entrypoint, copies its adjacent resources, and + bundles any non-public helper skills as one-level references so the + installed catalog skill remains self-contained. 2b. Optionally vendors the skill under a different local catalog name (the `as:` field on a skill entry). Federated skills follow a `-` naming convention in this catalog (e.g. the @@ -60,12 +65,12 @@ import re import shutil import subprocess -import sys import tempfile +import textwrap +from collections.abc import Iterable from dataclasses import dataclass from datetime import datetime, timezone from pathlib import Path -from typing import Iterable import yaml @@ -101,12 +106,26 @@ # description is still available in the vendored SKILL.md. MARKETPLACE_DESCRIPTION_MAX = 320 +# Exact public-wrapper body supported by `resolve_wrappers`. Requiring the +# whole body to match keeps resolution explicit and prevents prose from being +# interpreted as a filesystem instruction. +SKILL_POINTER_RE = re.compile( + r"\A\s*Read and follow the instructions in `(?P[^`]+/SKILL\.md)`\.\s*\Z" +) +# Runtime snippets in repository-native skills sometimes assume execution from +# the source checkout. Standalone catalog copies instead resolve resources from +# the loaded SKILL.md location. +SKILL_DIR_ASSIGNMENT_RE = re.compile( + r"(?m)^(?P[ \t]*)SKILL_DIR=\.claude/skills-impl/[^\s]+[ \t]*$" +) + @dataclass class SkillSpec: folder: str local_name: str | None = None marketplace_description_override: str | None = None + aliases: dict[str, str] | None = None @property def dest_name(self) -> str: @@ -122,6 +141,24 @@ class Source: path: str license: str skills: list[SkillSpec] + resolve_wrappers: bool = False + aliases: dict[str, str] | None = None + selected_skills: list[SkillSpec] | None = None + + @property + def import_skills(self) -> list[SkillSpec]: + return self.selected_skills if self.selected_skills is not None else self.skills + + +@dataclass +class ResolvedSkill: + spec: SkillSpec + public_dir: Path + implementation_dir: Path + public_path: str + implementation_path: str + public_frontmatter: dict + implementation_name: str @dataclass @@ -144,7 +181,7 @@ def parse_sources(catalog: Path) -> list[Source]: sources: list[Source] = [] for idx, raw in enumerate(raw_sources): if not isinstance(raw, dict): - raise ValueError(f"sources[{idx}] must be a mapping.") + raise TypeError(f"sources[{idx}] must be a mapping.") try: name = raw["name"] repo = raw["repo"] @@ -156,6 +193,17 @@ def parse_sources(catalog: Path) -> list[Source]: ) from None license_str = raw.get("license", "UNKNOWN") + source_aliases = raw.get("aliases") + if source_aliases is not None and ( + not isinstance(source_aliases, dict) + or not all( + isinstance(old, str) and isinstance(new, str) + for old, new in source_aliases.items() + ) + ): + raise ValueError( + f"sources[{idx}].aliases must be a string-to-string mapping." + ) skills_raw = raw.get("skills") or [] if not isinstance(skills_raw, list) or not skills_raw: raise ValueError( @@ -168,6 +216,18 @@ def parse_sources(catalog: Path) -> list[Source]: if isinstance(sk, str): skills.append(SkillSpec(folder=sk)) elif isinstance(sk, dict) and "name" in sk: + aliases = sk.get("aliases") + if aliases is not None and ( + not isinstance(aliases, dict) + or not all( + isinstance(old, str) and isinstance(new, str) + for old, new in aliases.items() + ) + ): + raise ValueError( + f"sources[{idx}].skills[{sk_idx}].aliases must be a " + "string-to-string mapping." + ) skills.append( SkillSpec( folder=sk["name"], @@ -175,6 +235,7 @@ def parse_sources(catalog: Path) -> list[Source]: marketplace_description_override=sk.get( "marketplace_description" ), + aliases=aliases, ) ) else: @@ -191,6 +252,8 @@ def parse_sources(catalog: Path) -> list[Source]: path=path.strip("/"), license=license_str, skills=skills, + resolve_wrappers=bool(raw.get("resolve_wrappers", False)), + aliases=source_aliases, ) ) return sources @@ -253,11 +316,13 @@ def _should_skip_target(target: str) -> bool: t = target.strip() if not t: return True - if t[0] in "#/": + # Regex examples such as `[A-Za-z](\w+)` are not Markdown links even + # though they match the lightweight link parser above. + if "\\" in t: return True - if URI_SCHEME_RE.match(t): + if t[0] in "#/": return True - return False + return bool(URI_SCHEME_RE.match(t)) def rewrite_external_references( @@ -280,50 +345,18 @@ def rewrite_external_references( (e.g. `reference.md`) are left untouched so they keep working locally. """ repo_skill_path = repo_skill_path.strip("/") - - def replace_in(text: str) -> tuple[str, list[tuple[str, str]]]: - rewrites: list[tuple[str, str]] = [] - - def _sub(match: re.Match[str]) -> str: - target = match.group("target") - if _should_skip_target(target): - return match.group(0) - path_part, sep, anchor = target.partition("#") - frag = sep + anchor if sep else "" - if not path_part: - return match.group(0) - - # Resolve the link both as the markdown spec would (relative to - # the file's folder in the repo) and relative to the repo root, - # since skill docs often write repo-root-relative paths. - skill_rel = posixpath.normpath(posixpath.join(repo_skill_path, path_part)) - root_rel = posixpath.normpath(path_part) - - within_skill = skill_rel == repo_skill_path or skill_rel.startswith( - repo_skill_path + "/" - ) - if within_skill and skill_rel in repo_files: - # Genuine intra-skill link; it was copied, leave it local. - return match.group(0) - - if skill_rel in repo_files: - chosen = skill_rel - else: - chosen = root_rel - - # Can't map something that points above the repo root. - if chosen.startswith("..") or chosen.startswith("/"): - return match.group(0) - - url = f"https://github.com/{repo}/blob/{commit}/{chosen}{frag}" - rewrites.append((target, url)) - return f"{match.group('prefix')}{url}{match.group('suffix')}" - - return MARKDOWN_LINK_RE.sub(_sub, text), rewrites - for md_path in sorted(skill_dir.rglob("*.md")): original = md_path.read_text(encoding="utf-8") - updated, rewrites = replace_in(original) + local_rel = md_path.relative_to(skill_dir).as_posix() + repo_file_path = posixpath.join(repo_skill_path, local_rel) + updated, rewrites = rewrite_external_references_in_text( + original, + repo_file_path=repo_file_path, + copied_repo_root=repo_skill_path, + repo_files=repo_files, + repo=repo, + commit=commit, + ) if updated != original: md_path.write_text(updated, encoding="utf-8") rel = md_path.relative_to(skill_dir.parent).as_posix() @@ -331,6 +364,52 @@ def _sub(match: re.Match[str]) -> str: log.append(f" [{rel}] {old} -> {new}") +def rewrite_external_references_in_text( + text: str, + *, + repo_file_path: str, + copied_repo_root: str | None, + repo_files: set[str], + repo: str, + commit: str, +) -> tuple[str, list[tuple[str, str]]]: + """Rewrite markdown links in one file using its original repo location.""" + rewrites: list[tuple[str, str]] = [] + repo_file_dir = posixpath.dirname(repo_file_path) + copied_repo_root = copied_repo_root.strip("/") if copied_repo_root else None + + def _sub(match: re.Match[str]) -> str: + target = match.group("target") + if _should_skip_target(target): + return match.group(0) + path_part, sep, anchor = target.partition("#") + frag = sep + anchor if sep else "" + if not path_part: + return match.group(0) + + # Resolve as Markdown does first, then try repo-root-relative because + # many product skills intentionally cite root paths such as docs/foo.md. + file_rel = posixpath.normpath(posixpath.join(repo_file_dir, path_part)) + root_rel = posixpath.normpath(path_part) + + if copied_repo_root: + within_copy = file_rel == copied_repo_root or file_rel.startswith( + copied_repo_root + "/" + ) + if within_copy and file_rel in repo_files: + return match.group(0) + + chosen = file_rel if file_rel in repo_files else root_rel + if chosen.startswith(("..", "/")): + return match.group(0) + + url = f"https://github.com/{repo}/blob/{commit}/{chosen}{frag}" + rewrites.append((target, url)) + return f"{match.group('prefix')}{url}{match.group('suffix')}" + + return MARKDOWN_LINK_RE.sub(_sub, text), rewrites + + def parse_frontmatter(text: str) -> dict: match = FRONTMATTER_RE.match(text) if not match: @@ -342,6 +421,319 @@ def parse_frontmatter(text: str) -> dict: return data if isinstance(data, dict) else {} +def _frontmatter_and_body(path: Path) -> tuple[dict, str]: + text = path.read_text(encoding="utf-8") + match = FRONTMATTER_RE.match(text) + if not match: + raise ValueError(f"{path} has no valid YAML frontmatter block.") + frontmatter = parse_frontmatter(text) + if not frontmatter: + raise ValueError(f"{path} has invalid or empty YAML frontmatter.") + return frontmatter, match.group("body") + + +def _repo_relative_path(clone_dir: Path, raw_path: str) -> tuple[Path, str]: + """Resolve and confine a wrapper target to the cloned repository.""" + normalized = posixpath.normpath(raw_path.strip().lstrip("/")) + if normalized.startswith("../") or normalized in {"", ".", ".."}: + raise ValueError(f"Unsafe wrapper target path: {raw_path!r}") + target = (clone_dir / normalized).resolve() + try: + target.relative_to(clone_dir.resolve()) + except ValueError as exc: + raise ValueError(f"Wrapper target escapes repository: {raw_path!r}") from exc + return target, normalized + + +def resolve_source_skills( + source: Source, + clone_dir: Path, + repo_files: set[str], +) -> list[ResolvedSkill]: + """Resolve configured public folders to the directories that get copied.""" + src_root = clone_dir / source.path + resolved: list[ResolvedSkill] = [] + for spec in source.skills: + public_dir = src_root / spec.folder + if not public_dir.is_dir(): + raise FileNotFoundError( + f"Skill {spec.folder!r} not found under " + f"{source.repo}/{source.path}@{source.ref}." + ) + public_md = public_dir / "SKILL.md" + if not public_md.exists(): + raise FileNotFoundError( + f"Skill {spec.folder!r} from {source.repo} has no SKILL.md." + ) + public_frontmatter, public_body = _frontmatter_and_body(public_md) + public_path = f"{source.path}/{spec.folder}".strip("/") + implementation_dir = public_dir + implementation_path = public_path + + if source.resolve_wrappers: + pointer = SKILL_POINTER_RE.fullmatch(public_body) + if pointer is None: + raise ValueError( + f"Skill {spec.folder!r} from {source.repo} must contain only " + "a supported `Read and follow .../SKILL.md` wrapper body when " + "`resolve_wrappers` is enabled." + ) + implementation_md, implementation_md_path = _repo_relative_path( + clone_dir, pointer.group("path") + ) + if ( + implementation_md_path not in repo_files + or not implementation_md.is_file() + ): + raise FileNotFoundError( + f"Wrapper target {implementation_md_path!r} for " + f"{spec.folder!r} does not exist in {source.repo}@{source.ref}." + ) + implementation_dir = implementation_md.parent + implementation_path = posixpath.dirname(implementation_md_path) + + implementation_frontmatter, _ = _frontmatter_and_body( + implementation_dir / "SKILL.md" + ) + implementation_name = implementation_frontmatter.get("name") + if not isinstance(implementation_name, str) or not implementation_name: + raise ValueError( + f"Implementation for {spec.folder!r} has no non-empty `name`." + ) + resolved.append( + ResolvedSkill( + spec=spec, + public_dir=public_dir, + implementation_dir=implementation_dir, + public_path=public_path, + implementation_path=implementation_path, + public_frontmatter=public_frontmatter, + implementation_name=implementation_name, + ) + ) + return resolved + + +def index_implementation_skills(source_root: Path) -> dict[str, Path]: + """Index non-wrapper skill implementations by frontmatter name.""" + index: dict[str, Path] = {} + for skill_md in sorted(source_root.rglob("SKILL.md")): + try: + frontmatter, body = _frontmatter_and_body(skill_md) + except ValueError: + continue + if SKILL_POINTER_RE.fullmatch(body): + continue + name = frontmatter.get("name") + if not isinstance(name, str) or not name: + continue + previous = index.get(name) + if previous is not None and previous != skill_md: + raise ValueError( + f"More than one implementation named {name!r}: " + f"{previous} and {skill_md}" + ) + index[name] = skill_md + return index + + +def _replace_skill_aliases(text: str, aliases: dict[str, str]) -> str: + for upstream_name, public_name in sorted( + aliases.items(), key=lambda item: len(item[0]), reverse=True + ): + if upstream_name != public_name: + text = re.sub( + rf"(? set[str]: + return { + name + for name in implementation_index + if re.search(rf"(? str: + text = SKILL_DIR_ASSIGNMENT_RE.sub( + lambda match: ( + f"{match.group('indent')}# Set SKILL_DIR to the directory containing " + "this installed SKILL.md." + ), + text, + ) + if implementation_path: + text = text.replace( + f"`{implementation_path}/`", "the installed skill directory" + ) + text = text.replace(f"`{implementation_path}`", "the installed skill directory") + return text.replace( + "Resolve `SKILL_DIR` from the repo root", + "Resolve `SKILL_DIR` from the loaded skill location", + ) + + +def bundle_supporting_skills( + skill_dir: Path, + *, + root_implementation_name: str, + implementation_index: dict[str, Path], + public_aliases: dict[str, str], + clone_dir: Path, + repo_files: set[str], + repo: str, + commit: str, + log: list[str], +) -> list[str]: + """Bundle non-public helper procedures referenced by a public skill.""" + root_parts: list[str] = [] + for path in sorted(skill_dir.rglob("*.md")): + text = path.read_text(encoding="utf-8") + if path == skill_dir / "SKILL.md": + match = FRONTMATTER_RE.match(text) + text = match.group("body") if match else text + root_parts.append(text) + root_text = "\n".join(root_parts) + public_implementations = set(public_aliases) + pending = sorted( + _mentioned_implementation_skills(root_text, implementation_index) + - public_implementations + - {root_implementation_name} + ) + bundled: dict[str, str] = {} + + while pending: + name = pending.pop(0) + if name in bundled: + continue + source_md = implementation_index[name] + text = source_md.read_text(encoding="utf-8") + text = _replace_skill_aliases(text, public_aliases) + repo_file_path = source_md.relative_to(clone_dir).as_posix() + text = _rewrite_standalone_runtime_paths( + text, posixpath.dirname(repo_file_path) + ) + text, rewrites = rewrite_external_references_in_text( + text, + repo_file_path=repo_file_path, + copied_repo_root=None, + repo_files=repo_files, + repo=repo, + commit=commit, + ) + bundled[name] = text + for old, new in rewrites: + log.append(f" [references/{name}.md] {old} -> {new}") + + dependencies = ( + _mentioned_implementation_skills(text, implementation_index) + - public_implementations + - {root_implementation_name, name} + - set(bundled) + ) + pending = sorted(set(pending) | dependencies) + + if not bundled: + return [] + references = skill_dir / "references" + references.mkdir(exist_ok=True) + for name, text in sorted(bundled.items()): + (references / f"{name}.md").write_text(text, encoding="utf-8") + log.append(f" bundled internal helper {name}") + return sorted(bundled) + + +def rewrite_frontmatter( + skill_dir: Path, + *, + name: str, + description: str, + log: list[str], +) -> None: + """Apply the public wrapper's routing metadata to a copied implementation.""" + skill_md = skill_dir / "SKILL.md" + text = skill_md.read_text(encoding="utf-8") + match = FRONTMATTER_RE.match(text) + if not match: + raise ValueError(f"Copied implementation {skill_md} has invalid frontmatter.") + description_lines = textwrap.wrap( + " ".join(description.split()), + width=96, + break_long_words=False, + break_on_hyphens=False, + ) + serialized = "\n".join( + [ + f"name: {name}", + "description: >-", + *[f" {line}" for line in description_lines], + ] + ) + skill_md.write_text( + f"---\n{serialized}\n---\n\n{match.group('body').lstrip()}", + encoding="utf-8", + ) + log.append(f" [SKILL.md] public routing metadata -> {name}") + + +def add_catalog_portability_section( + skill_dir: Path, + *, + source: Source, + commit: str, + bundled_helpers: list[str], +) -> None: + """Explain source-path and bundled-helper behavior in standalone installs.""" + skill_md = skill_dir / "SKILL.md" + text = skill_md.read_text(encoding="utf-8") + match = FRONTMATTER_RE.match(text) + if not match: + return + body = match.group("body") + section = [ + "## Catalog portability", + "", + ( + f"This standalone skill is federated from `{source.repo}` at commit `{commit}`. " + "Resolve bundled files relative to this `SKILL.md`. Repository-relative paths " + f"such as `docs/`, `examples/`, and `quark/` refer to the [pinned Quark source]" + f"(https://github.com/{source.repo}/tree/{commit}); use a local Quark checkout " + "when available, otherwise consult that pinned source." + ), + ] + if bundled_helpers: + section.extend( + [ + "", + ( + "When the workflow names one of these internal procedures, read its " + "bundled reference before carrying out that step:" + ), + "", + *[f"- [`{name}`](references/{name}.md)" for name in bundled_helpers], + ] + ) + section_text = "\n".join(section) + "\n\n" + first_heading = re.search(r"(?m)^# .+\n", body) + if first_heading: + insert_at = first_heading.end() + body = body[:insert_at] + "\n" + section_text + body[insert_at:].lstrip("\n") + else: + body = section_text + body + skill_md.write_text( + text[: match.start("body")] + body, + encoding="utf-8", + ) + + def truncate_description(text: str, limit: int = MARKETPLACE_DESCRIPTION_MAX) -> str: text = " ".join(text.split()) if len(text) <= limit: @@ -389,9 +781,7 @@ def copy_skill(src: Path, dest: Path, log: list[str] | None = None) -> None: for subdir, preserved_path in preserved.items(): shutil.copytree(preserved_path, dest / subdir) if log is not None: - log.append( - f" preserved local {subdir}/ (absent in upstream import)" - ) + log.append(f" preserved local {subdir}/ (absent in upstream import)") def write_marker( @@ -399,6 +789,7 @@ def write_marker( source: Source, commit: str, relative_path: str, + implementation_path: str | None = None, ) -> None: marker = { "source": source.name, @@ -409,6 +800,8 @@ def write_marker( "license": source.license, "imported_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), } + if implementation_path and implementation_path != relative_path: + marker["implementation_path"] = implementation_path (skill_dir / MARKER_FILENAME).write_text( json.dumps(marker, indent=2) + "\n", encoding="utf-8" ) @@ -426,7 +819,9 @@ def write_card(skill_dir: Path, source: Source, description: str) -> None: if card.exists(): return owner_org = source.repo.split("/")[0] - license_text = source.license or f"See [{source.repo}](https://github.com/{source.repo})" + license_text = ( + source.license or f"See [{source.repo}](https://github.com/{source.repo})" + ) card.write_text( "# Skill Card\n\n" "## Description\n\n" @@ -465,7 +860,9 @@ def rewrite_skill_name(skill_dir: Path, new_name: str, log: list[str]) -> None: new_frontmatter = f"name: {new_name}\n{frontmatter}" if new_frontmatter == frontmatter: return - skill_md.write_text(text[:fm_start] + new_frontmatter + text[fm_end:], encoding="utf-8") + skill_md.write_text( + text[:fm_start] + new_frontmatter + text[fm_end:], encoding="utf-8" + ) log.append(f" [SKILL.md] name -> {new_name}") @@ -538,25 +935,30 @@ def import_source( f"Path {source.path!r} not found in {source.repo}@{source.ref}." ) - for spec in source.skills: - src_skill = src_root / spec.folder - if not src_skill.is_dir(): - raise FileNotFoundError( - f"Skill {spec.folder!r} not found under " - f"{source.repo}/{source.path}@{source.ref}." - ) - skill_md = src_skill / "SKILL.md" - if not skill_md.exists(): - raise FileNotFoundError( - f"Skill {spec.folder!r} from {source.repo} has no SKILL.md." - ) - frontmatter = parse_frontmatter(skill_md.read_text(encoding="utf-8")) - description = frontmatter.get("description") or "" + resolved_skills = resolve_source_skills(source, tmp_path, repo_files) + implementation_index = ( + index_implementation_skills(src_root) if source.resolve_wrappers else {} + ) + public_aliases = { + resolved.implementation_name: resolved.spec.dest_name + for resolved in resolved_skills + } + selected_names = {spec.dest_name for spec in source.import_skills} + + for resolved in resolved_skills: + spec = resolved.spec + if spec.dest_name not in selected_names: + continue + skill_aliases = dict(public_aliases) + skill_aliases.update(source.aliases or {}) + skill_aliases.update(spec.aliases or {}) + description = resolved.public_frontmatter.get("description") or "" if not isinstance(description, str) or not description.strip(): raise ValueError( f"Skill {spec.folder!r} from {source.repo} has no " "non-empty `description` in its SKILL.md frontmatter." ) + description = _replace_skill_aliases(description, skill_aliases) marketplace_description = ( spec.marketplace_description_override or truncate_description(description) @@ -564,28 +966,81 @@ def import_source( dest_name = spec.dest_name dest_skill = SKILLS_DIR / dest_name - # The marker records the skill's *upstream* location, which keeps - # using the source folder name even when we vendor it locally as - # `dest_name`. - relative_path = f"{source.path}/{spec.folder}" action = "would import" if dry_run else "importing" renamed = f" (as {dest_name})" if dest_name != spec.folder else "" log.append( f"[{source.name}] {action} {spec.folder} -> skills/{dest_name}{renamed}" ) if not dry_run: - copy_skill(src_skill, dest_skill, log) - write_marker(dest_skill, source, commit, relative_path) - write_card(dest_skill, source, marketplace_description) - rewrite_skill_name(dest_skill, dest_name, log) + copy_skill(resolved.implementation_dir, dest_skill, log) + + if source.resolve_wrappers: + text_suffixes = { + ".md", + ".py", + ".json", + ".yaml", + ".yml", + ".rst", + ".txt", + } + for text_path in sorted(dest_skill.rglob("*")): + if ( + not text_path.is_file() + or text_path.suffix not in text_suffixes + ): + continue + text = text_path.read_text(encoding="utf-8") + text = _replace_skill_aliases(text, skill_aliases) + if text_path.suffix == ".md": + text = _rewrite_standalone_runtime_paths( + text, resolved.implementation_path + ) + text_path.write_text(text, encoding="utf-8") + rewrite_external_references( dest_skill, - relative_path, + resolved.implementation_path, repo_files, source.repo, commit, log, ) + if source.resolve_wrappers: + bundled_helpers = bundle_supporting_skills( + dest_skill, + root_implementation_name=resolved.implementation_name, + implementation_index=implementation_index, + public_aliases=skill_aliases, + clone_dir=tmp_path, + repo_files=repo_files, + repo=source.repo, + commit=commit, + log=log, + ) + rewrite_frontmatter( + dest_skill, + name=dest_name, + description=description.strip(), + log=log, + ) + add_catalog_portability_section( + dest_skill, + source=source, + commit=commit, + bundled_helpers=bundled_helpers, + ) + else: + rewrite_skill_name(dest_skill, dest_name, log) + + write_marker( + dest_skill, + source, + commit, + resolved.public_path, + resolved.implementation_path, + ) + write_card(dest_skill, source, marketplace_description) results.append( ImportResult( @@ -657,8 +1112,8 @@ def main(argv: list[str] | None = None) -> int: + ", ".join(sorted(unknown)) ) for source in sources: - source.skills = [s for s in source.skills if s.dest_name in only] - sources = [source for source in sources if source.skills] + source.selected_skills = [s for s in source.skills if s.dest_name in only] + sources = [source for source in sources if source.import_skills] log: list[str] = [] declared: set[str] = set() all_results: list[ImportResult] = [] @@ -667,7 +1122,7 @@ def main(argv: list[str] | None = None) -> int: existing_federated = find_federated_skills() for source in sources: - for spec in source.skills: + for spec in source.import_skills: if spec.dest_name in declared: raise ValueError( f"Skill name collision: {spec.dest_name!r} is listed by " @@ -691,7 +1146,7 @@ def main(argv: list[str] | None = None) -> int: for line in log: print(line) - print("") + print() print(f"Imported: {len(all_results)} skill(s)") print(f"Removed orphans: {len(pruned)}") print( diff --git a/.github/scripts/sources.yml b/.github/scripts/sources.yml index 5e6cfbc..219dc2b 100644 --- a/.github/scripts/sources.yml +++ b/.github/scripts/sources.yml @@ -14,16 +14,25 @@ # ref: branch, tag, or full commit SHA (pinned for reproducibility) # path: directory inside the repo that contains skill folders # license: SPDX license identifier (carried into the marker file) +# aliases: optional source-wide terminology corrections applied to every +# standalone copy; skill-entry aliases override these values +# resolve_wrappers: optional boolean. When true, each public skill folder +# must be a pointer to a repository-owned implementation +# SKILL.md. The importer produces a standalone catalog copy and +# bundles non-public helper procedures as references. # skills: list of skill folder names to import from `path` # Either a bare string (folder name) or a mapping: # { name: , as: , -# marketplace_description: } +# marketplace_description: , +# aliases: { : } } # `as` vendors the skill under a different local catalog name # (folder + marketplace entry + SKILL.md `name`). Federated # skills use a `-` convention, e.g. the # TraceLens `analysis-orchestrator` skill is imported `as: # tracelens-analysis-orchestrator`. The upstream `name` is # still used to locate the folder in the source repo. +# `aliases` applies narrow terminology corrections to the +# standalone copy without changing the product-owned source. # # To add or remove an imported skill, edit `skills:` here and then run the # "Import external skills" workflow. The workflow opens a pull request with @@ -46,3 +55,59 @@ sources: skills: - name: analysis-orchestrator as: tracelens-analysis-orchestrator + - name: amd-quark + repo: amd/Quark + ref: v0.12 + path: .claude + license: MIT + resolve_wrappers: true + aliases: + quark-onnx-doc-drift-check: the Quark repository's internal ONNX documentation drift checker + quark-onnx-router: the matching public quark-onnx skill + quark-onnx-skill-sync: the Quark repository's internal ONNX skill synchronizer + quark-torch-doc-drift-check: the Quark repository's internal Torch documentation drift checker + quark-torch-eval-runner: the Quark repository's internal skill evaluator + quark-torch-router: the matching public quark-torch skill + quark-torch-skill-sync: the Quark repository's internal Torch skill synchronizer + skills: + - name: skills/quark-env-preflight + as: quark-env-preflight + - name: skills/quark-install + as: quark-install + - name: skills/quark-onnx-autosearch-pro + as: quark-onnx-autosearch-pro + - name: skills/quark-onnx-debug + as: quark-onnx-debug + aliases: + quark-doc-drift-check: the Quark repository's internal ONNX documentation drift checker + quark-skill-sync: the Quark repository's internal ONNX skill synchronizer + - name: skills/quark-onnx-install + as: quark-onnx-install + - name: skills/quark-onnx-model-intake + as: quark-onnx-model-intake + - name: skills/quark-onnx-ptq + as: quark-onnx-ptq + - name: skills/quark-onnx-result-validator + as: quark-onnx-result-validator + - name: skills/quark-torch-debug + as: quark-torch-debug + - name: skills/quark-torch-export + as: quark-torch-export + aliases: + quark-onnx-export: quark-onnx-ptq + - name: skills/quark-torch-file2file-quantization + as: quark-torch-file2file-quantization + - name: skills/quark-torch-install + as: quark-torch-install + - name: skills/quark-torch-llm-eval + as: quark-torch-llm-eval + - name: skills/quark-torch-llm-ptq-eval + as: quark-torch-llm-ptq-eval + - name: skills/quark-torch-model-intake + as: quark-torch-model-intake + - name: skills/quark-torch-ptq + as: quark-torch-ptq + - name: skills/quark-torch-result-validator + as: quark-torch-result-validator + aliases: + quark-quantization-result-validator: quark-torch-result-validator diff --git a/.github/scripts/validate_quark_federation.py b/.github/scripts/validate_quark_federation.py new file mode 100755 index 0000000..9ae0365 --- /dev/null +++ b/.github/scripts/validate_quark_federation.py @@ -0,0 +1,185 @@ +#!/usr/bin/env -S uv run --quiet +# /// script +# requires-python = ">=3.10" +# dependencies = ["pyyaml>=6.0"] +# /// +"""Validate the standalone Quark family generated by federation.""" + +from __future__ import annotations + +import json +import re +from pathlib import Path + +import yaml + +REPO_ROOT = Path(__file__).resolve().parent.parent.parent +SOURCES_PATH = REPO_ROOT / ".github" / "scripts" / "sources.yml" +SKILLS_DIR = REPO_ROOT / "skills" +MARKETPLACE_PATH = REPO_ROOT / ".claude-plugin" / "marketplace.json" + +EXPECTED_SKILLS = { + "quark-env-preflight", + "quark-install", + "quark-onnx-autosearch-pro", + "quark-onnx-debug", + "quark-onnx-install", + "quark-onnx-model-intake", + "quark-onnx-ptq", + "quark-onnx-result-validator", + "quark-torch-debug", + "quark-torch-export", + "quark-torch-file2file-quantization", + "quark-torch-install", + "quark-torch-llm-eval", + "quark-torch-llm-ptq-eval", + "quark-torch-model-intake", + "quark-torch-ptq", + "quark-torch-result-validator", +} +LEGACY_NAMES = { + "quark-doc-drift-check", + "quark-onnx-export", + "quark-quantization-result-validator", + "quark-skill-sync", + "quark-torch-eval-runner", +} +SKILL_NAME_RE = re.compile(r"`(quark-[a-z0-9-]+)`") +POINTER_TEXT = "Read and follow the instructions in `.claude/skills-impl/" + + +def fail(errors: list[str], message: str) -> None: + errors.append(message) + + +def source_config(errors: list[str]) -> tuple[dict, dict[str, str]]: + data = yaml.safe_load(SOURCES_PATH.read_text(encoding="utf-8")) + matches = [ + source for source in data["sources"] if source.get("name") == "amd-quark" + ] + if len(matches) != 1: + fail(errors, "sources.yml must contain exactly one `amd-quark` source.") + return {}, {} + source = matches[0] + if source.get("repo", "").lower() != "amd/quark": + fail(errors, "The Quark source repository must be amd/Quark.") + if source.get("ref") != "v0.12": + fail(errors, "The initial Quark federation must remain pinned to v0.12.") + if source.get("resolve_wrappers") is not True: + fail(errors, "The Quark source must enable strict wrapper resolution.") + + configured: dict[str, str] = {} + for entry in source.get("skills", []): + if not isinstance(entry, dict) or not entry.get("as") or not entry.get("name"): + fail(errors, "Every Quark source skill must declare `name` and `as`.") + continue + configured[str(entry["as"])] = str(entry["name"]) + if set(configured) != EXPECTED_SKILLS: + fail( + errors, + "Configured Quark skills differ from the 17 public release skills: " + f"missing={sorted(EXPECTED_SKILLS - set(configured))}, " + f"extra={sorted(set(configured) - EXPECTED_SKILLS)}", + ) + return source, configured + + +def validate_skill( + name: str, + public_folder: str, + expected_ref: str, + errors: list[str], +) -> None: + skill_dir = SKILLS_DIR / name + skill_md = skill_dir / "SKILL.md" + marker_path = skill_dir / ".federated.json" + if not skill_md.is_file() or not marker_path.is_file(): + fail(errors, f"{name}: missing SKILL.md or .federated.json.") + return + + text = skill_md.read_text(encoding="utf-8") + if POINTER_TEXT in text: + fail(errors, f"{name}: unresolved Quark wrapper pointer remains.") + if "## Catalog portability" not in text: + fail(errors, f"{name}: missing standalone catalog portability guidance.") + + marker = json.loads(marker_path.read_text(encoding="utf-8")) + expected_public_path = f".claude/{public_folder}" + if marker.get("path") != expected_public_path: + fail(errors, f"{name}: marker path does not match {expected_public_path}.") + if marker.get("ref") != expected_ref: + fail(errors, f"{name}: marker ref does not match {expected_ref}.") + if not re.fullmatch(r"[0-9a-f]{40}", str(marker.get("commit", ""))): + fail(errors, f"{name}: marker is not pinned to a resolved commit SHA.") + implementation = str(marker.get("implementation_path", "")) + if not implementation.startswith(".claude/skills-impl/"): + fail(errors, f"{name}: marker does not record its resolved implementation.") + + bundled = {path.stem for path in (skill_dir / "references").glob("quark-*.md")} + all_markdown = "\n".join( + path.read_text(encoding="utf-8") for path in skill_dir.rglob("*.md") + ) + unresolved = set(SKILL_NAME_RE.findall(all_markdown)) - EXPECTED_SKILLS - bundled + if unresolved: + fail(errors, f"{name}: unresolved helper skill names: {sorted(unresolved)}") + stale = {legacy for legacy in LEGACY_NAMES if legacy in all_markdown} + if stale: + fail(errors, f"{name}: stale upstream skill names remain: {sorted(stale)}") + + +def validate_behavioral_contracts(errors: list[str]) -> None: + contracts = { + "quark-torch-ptq": ("safetensors", "quark-onnx-ptq", "explicit"), + "quark-onnx-ptq": (".onnx", "quark-torch-ptq", "approval"), + "quark-onnx-autosearch-pro": ("AutoSearchPro", "n_trials", "approval"), + } + for name, required in contracts.items(): + text = (SKILLS_DIR / name / "SKILL.md").read_text(encoding="utf-8") + missing = [phrase for phrase in required if phrase.lower() not in text.lower()] + if missing: + fail(errors, f"{name}: missing routing/safety contract terms {missing}.") + + for name, scripts in { + "quark-torch-result-validator": {"quant_validation.py", "run_validation.py"}, + "quark-onnx-result-validator": { + "quant_validation_onnx.py", + "run_validation.py", + }, + }.items(): + missing = [ + script for script in scripts if not (SKILLS_DIR / name / script).is_file() + ] + if missing: + fail(errors, f"{name}: missing bundled validator scripts {missing}.") + + +def validate_marketplace(errors: list[str]) -> None: + marketplace = json.loads(MARKETPLACE_PATH.read_text(encoding="utf-8")) + published = { + str(path).rstrip("/").split("/")[-1] + for path in marketplace["plugins"][0]["skills"] + } + missing = EXPECTED_SKILLS - published + if missing: + fail(errors, f"Marketplace does not publish Quark skills: {sorted(missing)}") + + +def main() -> int: + errors: list[str] = [] + source, configured = source_config(errors) + for name, public_folder in sorted(configured.items()): + validate_skill(name, public_folder, str(source.get("ref", "")), errors) + validate_behavioral_contracts(errors) + validate_marketplace(errors) + + if errors: + for error in errors: + print(f"[FAIL] {error}") + print(f"Quark federation validation failed with {len(errors)} error(s).") + return 1 + print(f"Quark federation validation passed for {len(EXPECTED_SKILLS)} skills.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/README.md b/README.md index 1f507aa..1a9d3aa 100644 --- a/README.md +++ b/README.md @@ -84,6 +84,32 @@ Cross-stack skills, from client to cloud. | `lemonade-router-builder` | Setup a Lemonade model router that handles requests based on content, sensitivity, or required capabilities. | _planned_ | | `hrr-replay-analysis` | Record, replay, and analyze GPU workload behavior on ROCm across AMD Instinct, Radeon, and Ryzen hardware using HIP Record and Replay archives. | _planned_ | +#### AMD Quark + +The Quark family keeps PyTorch/Hugging Face and ONNX artifacts on separate, +explicit paths while sharing environment and package setup. The catalog +federates the product-owned skills from the immutable Quark v0.12 release. + +| Skill | What it does | +| --- | --- | +| [`quark-env-preflight`](skills/quark-env-preflight/SKILL.md) | Inspect OS, Python, accelerator, CUDA/ROCm, and container readiness. | +| [`quark-install`](skills/quark-install/SKILL.md) | Install or verify AMD Quark and its core dependencies. | +| [`quark-torch-install`](skills/quark-torch-install/SKILL.md) | Install or verify the correct PyTorch build for the accelerator. | +| [`quark-torch-model-intake`](skills/quark-torch-model-intake/SKILL.md) | Inspect Hugging Face or safetensors model architecture and quantization risks. | +| [`quark-torch-ptq`](skills/quark-torch-ptq/SKILL.md) | Plan and run confirmed PyTorch/Hugging Face post-training quantization. | +| [`quark-torch-result-validator`](skills/quark-torch-result-validator/SKILL.md) | Validate exported safetensors and configuration artifacts. | +| [`quark-torch-debug`](skills/quark-torch-debug/SKILL.md) | Diagnose Torch, dependency, OOM, PTQ, and export failures. | +| [`quark-torch-export`](skills/quark-torch-export/SKILL.md) | Export quantized Torch models to Hugging Face, GGUF, or ONNX formats. | +| [`quark-torch-llm-eval`](skills/quark-torch-llm-eval/SKILL.md) | Evaluate LLM accuracy on ROCm with serving and evaluation frameworks. | +| [`quark-torch-llm-ptq-eval`](skills/quark-torch-llm-ptq-eval/SKILL.md) | Chain quantization, structural validation, and optional accuracy evaluation. | +| [`quark-torch-file2file-quantization`](skills/quark-torch-file2file-quantization/SKILL.md) | Quantize very large safetensors models without loading the full model. | +| [`quark-onnx-install`](skills/quark-onnx-install/SKILL.md) | Install or verify the correct ONNX Runtime provider build. | +| [`quark-onnx-model-intake`](skills/quark-onnx-model-intake/SKILL.md) | Inspect ONNX graphs, opsets, shapes, operators, and target compatibility. | +| [`quark-onnx-ptq`](skills/quark-onnx-ptq/SKILL.md) | Plan and run confirmed ONNX-to-ONNX post-training quantization. | +| [`quark-onnx-result-validator`](skills/quark-onnx-result-validator/SKILL.md) | Validate QDQ/custom-op insertion and unchanged ONNX initializers. | +| [`quark-onnx-debug`](skills/quark-onnx-debug/SKILL.md) | Diagnose ONNX Runtime, provider, calibration, and custom-op failures. | +| [`quark-onnx-autosearch-pro`](skills/quark-onnx-autosearch-pro/SKILL.md) | Run a bounded, confirmed Optuna search for an ONNX quantization configuration. | + ### Server-Native Run and optimize on AMD Instinct. diff --git a/docs/federating-your-repo.md b/docs/federating-your-repo.md index c95a78e..aea7686 100644 --- a/docs/federating-your-repo.md +++ b/docs/federating-your-repo.md @@ -18,6 +18,11 @@ this repo. - Skills live in a known directory in your repo (e.g. `skills/`). - Pick a branch to track (e.g. `main` or a release branch). +Product repositories may also keep lightweight public wrapper folders that +point to implementations elsewhere in the same repository. See +[Wrapper-based product skills](#wrapper-based-product-skills) for the stricter +format and standalone packaging behavior. + ## Add your source Edit [`.github/scripts/sources.yml`](../.github/scripts/sources.yml) and append an entry: @@ -36,6 +41,52 @@ sources: Use `as:` to namespace skills as `-` so catalog names stay unique. +## Wrapper-based product skills + +Set `resolve_wrappers: true` when the public skill folders contain only routing +frontmatter and an exact implementation pointer: + +```markdown +--- +name: my-product-build +description: Build a project with My Product. Use when ... +--- + +Read and follow the instructions in `.agents/skills-impl/my-product-build/SKILL.md`. +``` + +Configure `path` at a common ancestor of both the public wrappers and their +implementations, then address each wrapper relative to that path: + +```yaml + - name: amd-myproduct + repo: AMD-Org/MyProduct + ref: v1.0 + path: .agents + license: MIT + resolve_wrappers: true + skills: + - name: skills/my-product-build + as: myproduct-build +``` + +The importer uses the wrapper's `name` and `description` for routing, vendors +the implementation and its adjacent resources, rewrites escaping Markdown +links to the pinned upstream commit, and bundles referenced non-public helper +skills under `references/`. The generated catalog folder is therefore usable +when installed by itself; the wrapper is never shipped as a broken pointer. + +Wrapper resolution is deliberately strict: the body must contain exactly the +single `Read and follow .../SKILL.md` sentence, the target must be a tracked +file inside the same repository, and implementation skill names must be +unique beneath the configured source path. + +If a released implementation contains stale internal skill names, the source +or an individual skill entry may declare a narrow `aliases:` mapping from each +upstream term to its catalog wording. Skill-entry aliases override source-wide +values. Keep these corrections exceptional and remove them when the next +product release fixes the source. + ## Import Run the import scripts locally (they read `sources.yml` from your working tree), diff --git a/plugin-metadata.json b/plugin-metadata.json index d1f513c..8f4e4f1 100644 --- a/plugin-metadata.json +++ b/plugin-metadata.json @@ -14,6 +14,8 @@ "amd", "rocm", "hip", + "quark", + "quantization", "ryzen-ai", "vllm", "lemonade", diff --git a/skills/quark-env-preflight/.federated.json b/skills/quark-env-preflight/.federated.json new file mode 100644 index 0000000..01c9f37 --- /dev/null +++ b/skills/quark-env-preflight/.federated.json @@ -0,0 +1,10 @@ +{ + "source": "amd-quark", + "repo": "amd/Quark", + "ref": "v0.12", + "commit": "1b229f781a1974cc742884e42d8eefc1eebb4f0a", + "path": ".claude/skills/quark-env-preflight", + "license": "MIT", + "imported_at": "2026-08-05T21:07:39Z", + "implementation_path": ".claude/skills-impl/l0-foundation/shared/quark-env-preflight" +} diff --git a/skills/quark-env-preflight/SKILL.md b/skills/quark-env-preflight/SKILL.md new file mode 100644 index 0000000..ca27a97 --- /dev/null +++ b/skills/quark-env-preflight/SKILL.md @@ -0,0 +1,113 @@ +--- +name: quark-env-preflight +description: >- + Collect and normalize environment facts (OS, Python, GPU, CUDA/ROCm, container state) before + Quark installation or PTQ planning. Trigger for "check my environment", "what GPU do I have", + "is my setup ready for Quark", or when any accelerator-related assumption is unconfirmed. Also + trigger before any install/quantization step where hardware facts are missing. +--- + +# quark-env-preflight + +## Catalog portability + +This standalone skill is federated from `amd/Quark` at commit `1b229f781a1974cc742884e42d8eefc1eebb4f0a`. Resolve bundled files relative to this `SKILL.md`. Repository-relative paths such as `docs/`, `examples/`, and `quark/` refer to the [pinned Quark source](https://github.com/amd/Quark/tree/1b229f781a1974cc742884e42d8eefc1eebb4f0a); use a local Quark checkout when available, otherwise consult that pinned source. + +When the workflow names one of these internal procedures, read its bundled reference before carrying out that step: + +- [`quark-torch-quant-plan`](references/quark-torch-quant-plan.md) + +## Purpose + +Collect raw environment facts and normalize them into `env_context.json` so that downstream skills (install, model intake, PTQ planning) can make correct decisions without guessing. This skill is the single source of truth for hardware and toolchain state — getting it wrong here cascades into wrong install commands, incompatible packages, or failed quantization runs. + +## Inputs + +- None — runs standalone, reads from environment + +## Outputs: env_context.json + +Carries OS, Python, and hardware facts collected at preflight. + +Schema: [`env_context.schema.json`](https://github.com/amd/Quark/blob/1b229f781a1974cc742884e42d8eefc1eebb4f0a/.claude/skills-impl/shared/contracts/env_context.schema.json) + +```json +{ + "environment": { + "os": "linux", + "python": "3.13", + "containerized": false + }, + "hardware": { + "accelerator": "nvidia-cuda", + "cuda_version": "12.6", + "gpu_count": 1, + "gpu_model": "RTX 4090", + "memory_gb": 24 + } +} +``` + +`env_context.json` is for raw machine facts only — no installation results, no user goal, no open questions. Installed PyTorch and Quark versions live in `pytorch_install_result.json` and `quark_install_result.json` respectively. Unresolved questions belong in `session_context.json` (owned by `the matching public quark-torch skill`). + +## What to Detect + +### OS and Python + +- OS family and version (Linux distro, Windows version, WSL) +- Python version — Quark requires `>=3.11, <3.14` (pyproject.toml says `>=3.11`, setup.py says `>=3.9.0,<3.14`) +- Whether running inside conda/venv/virtualenv and the environment name +- Container state: Docker, Podman, or bare metal + +### Accelerator + +- **AMD ROCm**: check `ROCM_PATH`, `HIP_VISIBLE_DEVICES`, `rocm-smi` output, ROCm version (supported: 6.4, 7.0, 7.1) +- **NVIDIA CUDA**: check `CUDA_HOME`, `CUDA_VISIBLE_DEVICES`, `nvidia-smi` output, CUDA version (supported: 11.8, 12.6, 12.8, 13.0) +- **CPU-only**: only set `cpu` when the user explicitly requests CPU-only OR no GPU evidence exists after thorough checking +- Normalize to one of: `amd-rocm`, `nvidia-cuda`, `cpu`, `unknown` + +### Existing Quark Installation + +- Check `python -c "import quark; print(quark.__version__)"` — current version is 0.12 +- Check PyTorch version and its CUDA/ROCm build tag (`torch.version.cuda`, `torch.version.hip`) +- Check if `torch` and accelerator backend are from the same family (never mix CUDA torch with ROCm environment) + +## Detection Commands + +```bash +# OS and Python +python --version +uname -a # or systeminfo on Windows + +# GPU detection (try both, one will fail gracefully) +nvidia-smi --query-gpu=name,memory.total,driver_version --format=csv,noheader 2>/dev/null +rocm-smi --showproductname 2>/dev/null + +# Environment variables +echo $CUDA_HOME $CUDA_VISIBLE_DEVICES $ROCM_PATH $HIP_VISIBLE_DEVICES + +# Existing packages +pip show amd-quark torch 2>/dev/null +python -c "import torch; print(torch.__version__, torch.version.cuda, torch.version.hip)" +``` + +## Rules + +- **Only collect and normalize facts.** Do not choose install commands, PTQ schemes, or workflow branches — that is the job of `quark-install` or `quark-torch-quant-plan`. +- **Prefer explicit user input** over heuristic inference when they conflict. If the user says "I'm on ROCm" but `CUDA_HOME` is also set, trust the user. +- **Missing evidence ≠ CPU.** Treat missing `CUDA_VISIBLE_DEVICES`, `ROCM_PATH`, or `HIP_VISIBLE_DEVICES` as insufficient evidence for `cpu`. Keep the accelerator as `unknown` until something definitive is found. +- **Never upgrade `unknown` to `cpu`** unless the user explicitly says CPU-only or detection confirms zero GPU hardware. +- **Capture version dependencies** even if the exact version is not yet known. For example, if the user mentions "ROCm" but not the version, record `accelerator: amd-rocm, rocm_version: unknown`. + +## Interaction Flow + +1. **Intake**: Ask what downstream task the user is headed toward (install? PTQ? just checking?). This determines which facts are critical vs. nice-to-have. +2. **Detect**: Run the detection commands above. Present what was found in a clear summary table. +3. **Clarify**: If the accelerator is ambiguous or versions are uncertain, ask the user — do not guess. Show them the conflicting evidence. +4. **Emit**: Write the confirmed facts to `env_context.json`. Hand any unresolved items back to the caller (typically `the matching public quark-torch skill`) so they land in `session_context.json` under `open_questions`. + +## Recovery + +- If environment evidence is contradictory (e.g., both CUDA and ROCm libraries present), keep both raw facts in the summary, set `accelerator=unknown`, and explain the conflict. +- If a detection command cannot run (e.g., no permissions for `nvidia-smi`), report exactly which signal is blocked and suggest the smallest manual check: "Run `nvidia-smi` in a terminal with GPU access and paste the output." +- If Python version is outside `3.11–3.13`, flag it immediately — Quark will not work. diff --git a/skills/quark-env-preflight/references/quark-torch-quant-plan.md b/skills/quark-env-preflight/references/quark-torch-quant-plan.md new file mode 100644 index 0000000..2694f3b --- /dev/null +++ b/skills/quark-env-preflight/references/quark-torch-quant-plan.md @@ -0,0 +1,209 @@ +--- +name: quark-torch-quant-plan +description: > + Build a Quark Torch LLM PTQ quantization plan from model analysis and user intent. Use when the + user needs quantization scheme recommendations, exclusion lists, algorithm selection, KV cache + decisions, per-layer overrides, or a draft quant_plan. Trigger for "quantize with FP8", + "what scheme should I use", "plan PTQ", "INT4 quantization", "choose quantization config", + "quantization plan", or when the user has a model analysis and needs to decide how to quantize it. +layer: l1-atomic +primary_artifact: quant_plan.json +source_knowledge: + - quark/torch/quantization/config/template.py + - examples/torch/language_modeling/llm_ptq/README.md + - examples/torch/language_modeling/llm_ptq/quantize_quark.py +--- + +# quark-torch-quant-plan + +## Purpose + +Convert a model analysis plus the user's intent into a confirmed `quant_plan.json`. This skill makes the quantization decisions — which scheme, which algorithm, what to exclude — without generating scripts or executing PTQ. The plan is the contract between the user's intent and the execution step. + +## Inputs + +- `model_analysis.json` from quark-torch-model-intake +- `env_context.json` for accelerator-aware scheme recommendations +- User preferences (target precision, accuracy goal) + +## Outputs: quant_plan.json + +Records the chosen scheme, algorithm, layer overrides, calibration settings, and evaluation intent. + +Schema: [`quant_plan.schema.json`](https://github.com/amd/Quark/blob/1b229f781a1974cc742884e42d8eefc1eebb4f0a/.claude/skills-impl/shared/contracts/quant_plan.schema.json) + +```json +{ + "model": { + "model_type": "qwen3", + "analysis_ref": "./model_analysis.json" + }, + "global_scheme": "fp8", + "kv_cache_scheme": "fp8", + "exclude_layers": ["lm_head"], + "layer_quant_config": {}, + "algorithm": null, + "calibration": { + "dataset": "pileval", + "num_calib_data": 128, + "seq_len": 512 + }, + "evaluation_intent": "smoke", + "requires_confirmation": false +} +``` + +## Available Quantization Schemes (21 total) + +### Weight-Only INT4 (best for deployment size reduction) + +| Scheme | Description | Use Case | +|--------|-------------|----------| +| `int4_wo_32` | INT4, group size 32 | Highest accuracy among INT4 | +| `int4_wo_64` | INT4, group size 64 | Good balance | +| `int4_wo_128` | INT4, group size 128 | Smaller overhead | +| `int4_wo_per_channel` | INT4, per-channel | Least overhead | +| `uint4_wo_32/64/128/per_channel` | Unsigned INT4 variants | GGUF export compatibility | + +### Weight+Activation INT8 + +| Scheme | Description | Use Case | +|--------|-------------|----------| +| `int8` | INT8 per-tensor for both W and A | CPU deployment, good accuracy | + +### FP8 (best accuracy-size tradeoff for GPU inference) + +| Scheme | Description | Use Case | +|--------|-------------|----------| +| `fp8` | FP8 E4M3 per-tensor | Standard GPU quantization | +| `ptpc_fp8` | Per-Token-Per-Channel FP8 | Higher accuracy, dynamic activation quantization | + +### OCP Microscaling Formats + +| Scheme | Description | Use Case | +|--------|-------------|----------| +| `mxfp4` | OCP MXFP4 | Aggressive compression | +| `mxfp6_e3m2` | OCP MXFP6 (E3M2) | Better range | +| `mxfp6_e2m3` | OCP MXFP6 (E2M3) | Better precision | +| `mxfp4_mxfp6_e2m3` | MXFP4 weights + MXFP6 activations | Mixed precision | +| `mxfp4_fp8` | MXFP4 weights + FP8 activations | Mixed precision | + +### AMD-Specific + +| Scheme | Description | Use Case | +|--------|-------------|----------| +| `amdfp4` | amdfp4, group size 16 | AMD MI300X optimized | +| `amdfp4_g32` | amdfp4, group size 32 | AMD MI300X, less overhead | + +### Other + +| Scheme | Description | Use Case | +|--------|-------------|----------| +| `nvfp4` | NVFP4: FP4 group_size=16 with FP8 E4M3 scale | NVIDIA Blackwell/Hopper | +| `mx6` | MX6 format | Experimental | +| `bfp16` | Block Floating Point 16-bit | Experimental | +| `int4_wa_64` | INT4 weights + activations, group 64 | Research | + +## Available Algorithms (7 primary) + +| Algorithm | Compatible Schemes | Description | +|-----------|-------------------|-------------| +| `awq` | INT4/UINT4 weight-only | Activation-aware weight quantization — finds optimal per-channel scaling | +| `gptq` | INT4/UINT4 weight-only | Second-order weight optimization — often better than AWQ for small models | +| `smoothquant` | INT8, FP8 | Migrates quantization difficulty from activations to weights | +| `autosmoothquant` | INT8, FP8 | Automatic SmoothQuant with optimal alpha search | +| `rotation` | Various | Rotation-based optimization to equalize weight distribution | +| `gptaq` | INT4/UINT4 | GPTAQ variant combining GPTQ with activation quantization | +| `qronos` | Various | Custom algorithm for time-series-aware quantization | + +Algorithms can be combined: `--quant_algo awq,smoothquant` + +## KV Cache Quantization + +- Only `fp8` is supported for KV cache (`--kv_cache_dtype fp8`) +- Adds `--min_kv_scale` option (default 0.0) to prevent extreme scale values +- `--kv_cache_post_rope` quantizes KV cache after RoPE (inside cache) instead of at k_proj/v_proj outputs — can improve accuracy for some models + +## Decision Guide + +Help the user choose based on their priorities: + +**"I want the best accuracy"** → `fp8` or `ptpc_fp8`, optionally with `smoothquant` +**"I want the smallest model"** → `int4_wo_32` with `awq` or `gptq` +**"I need CPU deployment"** → `int8` (the only scheme that works well on CPU) +**"I need GGUF format"** → `uint4_wo_32` with `awq`, export as GGUF +**"I'm on AMD MI300X"** → `amdfp4` for best hardware utilization +**"I'm on NVIDIA H100/Blackwell"** → `fp8` or `nvfp4` +**"I want to experiment"** → `mxfp4` for aggressive compression research + +## Decision Table (MUST show to user) + +**ALWAYS present this table to the user and WAIT for confirmation before finalizing.** Do not skip this step. + +Fill in the "Value" column based on the user's request and model analysis, then show: + +| Decision | Value | Reason | +|----------|-------|--------| +| `global_scheme` | _(fill)_ | _(why this scheme)_ | +| `kv_cache_scheme` | _(fill: `fp8` or `null`)_ | _(explain)_ | +| `exclude_layers` | `["lm_head"]` | Standard — lm_head stays full precision | +| `layer_quant_config` | _(fill: dict of `pattern -> scheme`, or `{}` if none)_ | _(explain which patterns and why)_ | +| `algorithm` | _(fill: algorithm or `null`)_ | _(explain)_ | +| `calibration_dataset` | `pileval` | Fast default | +| `num_calib_data` | `128` | Standard default | +| `seq_len` | `512` | Standard default | +| `evaluation_intent` | `smoke` | Quick PPL check post-quantization | + +After showing the table, ask: "Confirm this plan? Any changes?" + +**Do NOT proceed until the user confirms.** + +## Layer-Specific Overrides via `layer_quant_config` + +The `layer_quant_config` plan field is a dict of `pattern -> scheme` pairs. It is the +single mechanism for any "quantize layer/module X with scheme Y" intent — including +attention modules, MoE experts, lm_head, etc. Each entry emits one +`--layer_quant_scheme PATTERN SCHEME` CLI argument. + +```json +"layer_quant_config": { + "*self_attn*": "fp8", + "lm_head": "int8", + "*experts*": "fp8" +} +``` + +translates to: + +```bash +--quant_scheme \ +--layer_quant_scheme '*self_attn*' fp8 \ +--layer_quant_scheme lm_head int8 \ +--layer_quant_scheme '*experts*' fp8 +``` + +When a user asks for attention-module quantization (e.g. "self_attn in fp8"), populate +this field with the appropriate pattern (commonly `*self_attn*` for LLaMA-style models; +adjust for models whose attention submodule has a different name). Do NOT introduce a +dedicated attention field — keep all per-pattern overrides in `layer_quant_config`. + +## Rules + +- **Keep scope to plan creation only.** Do not generate scripts, do not run quantization, do not export. Those are separate skills. +- **Require a model analysis first.** Without knowing the model architecture and layer count, you cannot make informed scheme recommendations. If `model_analysis.json` is missing, route back to `quark-torch-model-intake`. +- **Always present the decision table** before finalizing. The user should explicitly confirm the scheme, algorithm, and exclusions. +- **If a risky scheme is chosen** (e.g., `mxfp4` on a model where accuracy loss may be significant), keep the user's choice but record the risk in the plan. + +## Interaction Flow + +1. **Check prerequisites**: Is `model_analysis.json` available? If not, route to `quark-torch-model-intake` first. +2. **Gather intent**: What does the user care about most — accuracy, size, speed? What hardware will run inference? +3. **Present the decision table**: Show defaults, explain the tradeoffs, and let the user adjust. +4. **Confirm**: Always required. Show the final plan summary before writing it. +5. **Emit**: Write `quant_plan.json`. + +## Recovery + +- If the model analysis is incomplete, produce a draft plan with `requires_confirmation: true` and note what facts are missing. +- If the user picks an unusual combination (e.g., `awq` with `fp8` — AWQ is designed for INT4), explain why it might not work well and suggest alternatives, but respect the user's choice if they insist. +- If calibration dataset preferences are unclear, default to `pileval` with 128 samples — it is the fastest option and works for most models. diff --git a/skills/quark-env-preflight/skill-card.md b/skills/quark-env-preflight/skill-card.md new file mode 100644 index 0000000..c3a4a6b --- /dev/null +++ b/skills/quark-env-preflight/skill-card.md @@ -0,0 +1,13 @@ +# Skill Card + +## Description + +Collect and normalize environment facts (OS, Python, GPU, CUDA/ROCm, container state) before Quark installation or PTQ planning. Trigger for "check my environment", "what GPU do I have", "is my setup ready for Quark", or when any accelerator-related assumption is unconfirmed. + +## Owner + +amd (federated from [amd/Quark](https://github.com/amd/Quark)) + +## License + +MIT diff --git a/skills/quark-install/.federated.json b/skills/quark-install/.federated.json new file mode 100644 index 0000000..d9b8b82 --- /dev/null +++ b/skills/quark-install/.federated.json @@ -0,0 +1,10 @@ +{ + "source": "amd-quark", + "repo": "amd/Quark", + "ref": "v0.12", + "commit": "1b229f781a1974cc742884e42d8eefc1eebb4f0a", + "path": ".claude/skills/quark-install", + "license": "MIT", + "imported_at": "2026-08-05T21:07:39Z", + "implementation_path": ".claude/skills-impl/l1-atomic/shared/quark-install" +} diff --git a/skills/quark-install/SKILL.md b/skills/quark-install/SKILL.md new file mode 100644 index 0000000..d6e2ad4 --- /dev/null +++ b/skills/quark-install/SKILL.md @@ -0,0 +1,165 @@ +--- +name: quark-install +description: >- + Install or verify the AMD Quark package and its dependencies. Trigger for "install Quark", "set + up Quark", "pip install amd-quark", dependency errors, import failures for quark modules, + ModuleNotFoundError for quark, or missing C++ compiler errors. For PyTorch installation or torch + version issues, use quark-torch-install instead. +--- + +# quark-install + +## Catalog portability + +This standalone skill is federated from `amd/Quark` at commit `1b229f781a1974cc742884e42d8eefc1eebb4f0a`. Resolve bundled files relative to this `SKILL.md`. Repository-relative paths such as `docs/`, `examples/`, and `quark/` refer to the [pinned Quark source](https://github.com/amd/Quark/tree/1b229f781a1974cc742884e42d8eefc1eebb4f0a); use a local Quark checkout when available, otherwise consult that pinned source. + +## Purpose + +Install the AMD Quark package and its dependencies after PyTorch is already set up. This skill handles Quark-specific setup: the `amd-quark` package, core dependencies, optional ONNX Runtime, LLM PTQ extras, and compiler requirements. It exists separately from `quark-torch-install` (which handles PyTorch) and from PTQ planning because getting the environment right is a prerequisite — a missing dependency or wrong compiler will cause cryptic failures later. + +## Inputs + +- `env_context.json` for OS/Python/accelerator facts +- `pytorch_install_result.json` confirming PyTorch is installed and verified + +## Outputs: quark_install_result.json + +Records the installed Quark version, optional extras (ONNX runtime, LLM PTQ deps), and verification status. + +Schema: [`quark_install_result.schema.json`](https://github.com/amd/Quark/blob/1b229f781a1974cc742884e42d8eefc1eebb4f0a/.claude/skills-impl/shared/contracts/quark_install_result.schema.json) + +```json +{ + "status": "ok", + "quark_version": "0.12", + "install_source": "pypi", + "extras_installed": { + "onnxruntime": false, + "llm_ptq_deps": true + }, + "verification": { + "import_ok": true, + "kernel_ok": true, + "onnx_ops_ok": null + } +} +``` + +On failure, set `status: "failed"` and include a `failure_reason` with the exact failing verification command. + +## Quark Package Info + +- **PyPI package**: `amd-quark` (current version: 0.12) +- **Install from PyPI (universal wheel, recommended default)**: `pip install amd-quark`. Works on any OS/Python/accelerator regardless of PyTorch version, but compiles the fast quantization kernels and ONNX custom-op library on first import (requires a C++ compiler, plus `nvcc`/`hipcc` for GPU). +- **Install a pre-built wheel (optional, PyTorch 2.10+)**: ships pre-compiled C++ extensions, so no C++ compiler and no first-run compilation are needed. Hosted on the AMD package index (Python 3.11–3.13); point `pip` at the matching index: + + ```bash + pip install amd-quark --extra-index-url https://pypi.amd.com/quark/cpu/simple # CPU + pip install amd-quark --extra-index-url https://pypi.amd.com/quark/cu128/simple # CUDA 12.8 + pip install amd-quark --extra-index-url https://pypi.amd.com/quark/rocm71/simple # ROCm 7.1, Linux only + pip install amd-quark --extra-index-url https://pypi.amd.com/quark/rocm72/simple # ROCm 7.2, Linux only + ``` + +- **Install from source**: + + ```bash + git clone --recursive https://github.com/AMD/Quark + cd Quark + git submodule sync && git submodule update --init --recursive + pip install . + ``` + +- **Install from wheel**: `pip install amd_quark*.whl` + +## Python Version Requirements + +- **Supported**: Python 3.11, 3.12, 3.13 +- **Not supported**: Python 3.14+ +- **Recommended for new setups**: Python 3.13 via Miniforge/Miniconda + +## ONNX Runtime (Optional) + +- Version constraint: `>=1.22.2, <=1.24.2` +- GPU variant: `pip install onnxruntime-gpu` (for CUDA) +- CPU variant: `pip install onnxruntime` +- ROCm note: use the CPU variant of ONNX Runtime for ROCm 7.0+ due to build compatibility issues + +## LLM PTQ Additional Dependencies + +For running `quantize_quark.py`, install these extras: + +```bash +pip install accelerate datasets evaluate>=0.4.0 gguf>=0.10.0 lm-eval transformers<5.3 +``` + +## Core Dependencies (from requirements.txt) + +```text +evaluate, joblib, ninja, numpy>=2.0, onnx>=1.21.0,<=1.22.0, onnxscript, +onnxslim>=0.1.84, pandas, plotly, protobuf, psutil, pydantic, rich, scipy, +sentencepiece, tqdm, zstandard +``` + +## Compiler Requirements + +- **Linux**: `sudo apt install build-essential` (includes g++, needed for kernel compilation) +- **Windows**: Visual Studio 2022+ with "Desktop development with C++" workload + +## Rules + +- **Ensure PyTorch is already installed and verified.** If PyTorch is missing or mismatched with the accelerator, hand off to `quark-torch-install` first. Do not attempt to install Quark without a working PyTorch. +- **Never skip verification.** After installation, always run verification commands. +- **Show exact commands before execution.** The user should see every `pip install` command and every version before anything runs. + +## Verification Commands + +```bash +# Basic import +python -c "import quark; print('Quark version:', quark.__version__)" + +# Optional: kernel compilation test +python -c "import quark.torch.kernel; print('Kernel compilation OK')" + +# Optional: ONNX custom ops +python -c "import quark.onnx.operators.custom_ops; print('ONNX custom ops OK')" +``` + +## Interaction Flow + +1. **Intake**: Determine what the user already has installed and what they need. Check if `quark-torch-install` has already run and PyTorch is verified. +2. **Plan**: Present the installation plan as a numbered sequence of commands, with version justifications. +3. **Confirm**: Required before any package installation. Show: what will be installed and what environment will be modified. +4. **Execute**: Run the installation commands. +5. **Verify**: Run all verification commands. Report pass/fail for each. + +## Recovery + +- **If verification fails**: Show the exact failing check and the most likely cause. Common issues: + - `ModuleNotFoundError: No module named 'quark'` — Quark not installed or wrong Python environment + - `ImportError: quark.torch.kernel` — Missing `build-essential` / C++ compiler +- **If PyTorch is missing or mismatched**: Hand off to `quark-torch-install` with the specific issue noted. Do not attempt to fix PyTorch issues from this skill. +- **If Python version is wrong**: Recommend creating a new conda environment with a supported version. + +## Windows-Specific Notes + +- If pip fails with long path errors: Enable Win32 long paths via Group Policy Editor (Computer Configuration > Administrative Templates > System > Filesystem > Enable Win32 long paths) +- WSL2 with Ubuntu is recommended as an alternative for Windows users +- ROCm is not supported on Windows — only CUDA and CPU + +## Docker Option + +Quark provides official Dockerfiles for reproducible environments: + +- `Dockerfile.cuda` — NVIDIA CUDA (base image: `nvidia/cuda:11.8.0-base-ubuntu22.04`) +- `Dockerfile.rocm` — AMD ROCm (base image: `rocm/dev-ubuntu-24.04:6.4`) +- `Dockerfile.cpu` — CPU only (base image: `ubuntu:22.04`) + +Build with: + +```bash +docker build -f tools/ci/docker/images/Dockerfile.cuda \ + --build-arg PYTHON_VERSION=3.13 \ + --build-arg PYTORCH_VERSION=2.10.0 \ + --build-arg ACCELERATOR_VERSION=cuda-12.6 \ + -t quark:cuda . +``` diff --git a/skills/quark-install/skill-card.md b/skills/quark-install/skill-card.md new file mode 100644 index 0000000..3a87b4a --- /dev/null +++ b/skills/quark-install/skill-card.md @@ -0,0 +1,13 @@ +# Skill Card + +## Description + +Install or verify the AMD Quark package and its dependencies. Trigger for "install Quark", "set up Quark", "pip install amd-quark", dependency errors, import failures for quark modules, ModuleNotFoundError for quark, or missing C++ compiler errors. + +## Owner + +amd (federated from [amd/Quark](https://github.com/amd/Quark)) + +## License + +MIT diff --git a/skills/quark-onnx-autosearch-pro/.federated.json b/skills/quark-onnx-autosearch-pro/.federated.json new file mode 100644 index 0000000..a6856b4 --- /dev/null +++ b/skills/quark-onnx-autosearch-pro/.federated.json @@ -0,0 +1,10 @@ +{ + "source": "amd-quark", + "repo": "amd/Quark", + "ref": "v0.12", + "commit": "1b229f781a1974cc742884e42d8eefc1eebb4f0a", + "path": ".claude/skills/quark-onnx-autosearch-pro", + "license": "MIT", + "imported_at": "2026-08-05T21:07:39Z", + "implementation_path": ".claude/skills-impl/l3-recipes/onnx/quark-onnx-autosearch-pro" +} diff --git a/skills/quark-onnx-autosearch-pro/SKILL.md b/skills/quark-onnx-autosearch-pro/SKILL.md new file mode 100644 index 0000000..076aa55 --- /dev/null +++ b/skills/quark-onnx-autosearch-pro/SKILL.md @@ -0,0 +1,304 @@ +--- +name: quark-onnx-autosearch-pro +description: >- + End-to-end Quark ONNX AutoSearchPro recipe — drives `quark.onnx.AutoSearchPro` (Optuna-based + hyperparameter search) on a `.onnx` model to find the best quantization config + (activation/weight spec, calibration method, CLE, AdaRound / AdaQuant, FastFinetune params). Use + when the user wants to "auto search", "tune quantization", "find the best quant config", "sweep + AdaRound/AdaQuant", "run AutoSearchPro / AutoSearch", "two-stage search", or pick one of the + built-in presets (`ADVANCED_SEARCH`, `XINT8_SEARCH`, `A8W8_SEARCH`, `A16W8_SEARCH`) for their + `.onnx` model. Not for HuggingFace / safetensors / PyTorch input models — use quark-torch-ptq + instead. Not for single-shot ONNX PTQ without search — use quark-onnx-ptq. +--- + +# quark-onnx-autosearch-pro + +## Catalog portability + +This standalone skill is federated from `amd/Quark` at commit `1b229f781a1974cc742884e42d8eefc1eebb4f0a`. Resolve bundled files relative to this `SKILL.md`. Repository-relative paths such as `docs/`, `examples/`, and `quark/` refer to the [pinned Quark source](https://github.com/amd/Quark/tree/1b229f781a1974cc742884e42d8eefc1eebb4f0a); use a local Quark checkout when available, otherwise consult that pinned source. + +## Purpose + +Drive Quark ONNX `AutoSearchPro` (Optuna-driven) on a `.onnx` model to discover +the best quantization configuration — activation / weight specs, calibration +method, CLE, AdaRound / AdaQuant hyperparameters — without hand-tuning. The +recipe orchestrates intake, preset (or custom) search-space selection, reader +construction, generation of a standalone autosearch script in the user's +workspace, confirmed execution, and reporting of `best_params.json` plus +Optuna study artifacts. Unlike single-shot PTQ this produces many candidate +models, so preset and trial budget are explicit and the search is bounded. + +## Inputs + +- Input `.onnx` model (with optional sibling `.onnx_data`). +- Calibration data: folder of representative samples or a Python + `CalibrationDataReader` class. Optional separate evaluation reader. +- User goal: preset choice (`ADVANCED_SEARCH` / `XINT8_SEARCH` / + `A8W8_SEARCH` / `A16W8_SEARCH` / custom dict), deployment target, search + budget (`n_trials`, `n_jobs`, `two_stage_search`), and metric (built-in or + custom callable). +- `session_context.json` (`constraints.backend = "onnx"`), `env_context.json`, + `workspace_context.json`, plus `onnx_install_result.json` and + `quark_install_result.json`. **Optuna** must be importable + (`pip install optuna`). + +## Outputs: run_manifest.yaml + +Records the generated autosearch script path, the exact `python3` invocation, +the preset name (or custom search-space hash), trial budget, Optuna study DB +path, and the final `best_params.json` location. + +Side artifacts written by `AutoSearchPro` into `output_dir`: +`auto_search.log`, `best_params.json`, `auto_search.db` (resumable Optuna +study), `quantized_model_.onnx` per trial, and `opt_history.html` / +`param_importance.html` when `plot_results=True`. + +Built in Step 4. Schema: +[`run_manifest.schema.json`](https://github.com/amd/Quark/blob/1b229f781a1974cc742884e42d8eefc1eebb4f0a/.claude/skills-impl/shared/contracts/run_manifest.schema.json) + +## Interaction Flow + +1. **Intake** — `quark-onnx-model-intake` → `model_analysis.json` +2. **Preset / search space** — pick preset or build custom → `quant_plan.json` + (with `search_space` block + `n_trials`) +3. **Reader + evaluator** — confirm `CalibrationDataReader` and evaluator mode +4. **Manifest** — generate standalone autosearch script in user workspace + + `run_manifest.yaml`; stop for user approval +5. **Execute** — run the confirmed script, report `best_params.json` + +## CRITICAL RULES + +1. **NEVER call `AutoSearchPro(...).run()` directly from this skill.** Generate + a standalone script the user reviews first. Searches can take hours and + write many `.onnx` files — user must opt in. +2. **NEVER skip preset confirmation.** Presets default to + `optim_device = "cuda:0"`; surface device gaps before execution. +3. **NEVER silently fall back to CPU** when CUDA/ROCm was requested. Hand off + to `quark-onnx-install` / `quark-onnx-debug`. +4. **NEVER write into the Quark repo.** Generated scripts go to the user's + workspace, *not* under `examples/onnx/auto_search/` or + `tutorials/onnx/auto_search/`. The shipped + `examples/onnx/auto_search/auto_search_pro_model.py` is a template to read, + not patch. +5. **Pre-check optuna.** `AutoSearchPro` imports it lazily; verify it imports + in the user's env before writing the script. +6. **Bound the budget.** Refuse `n_trials > 200` without explicit user opt-in + — one `.onnx` per trial. +7. **Validate custom search spaces** with + `quark.onnx.quantization.auto_search.utils.validate_search_space`. If + `n_trials > discrete_space_size` without continuous fields, AutoSearchPro + auto-clamps — surface it instead of letting it happen quietly. + +See [`presets-reference.md`](presets-reference.md) for the preset table, +deployment recommendations, custom-search-space rules, device gating snippet, +evaluator modes, and sampler list. + +## Required Artifact Flow + +```text +Step 1 (Intake) ──► model_analysis.json +Step 2 (Preset/space) ──► quant_plan.json (search_space + n_trials) +Step 3 (Reader) ──► (informs script body) +Step 4 (Manifest) ──► run_manifest.yaml + user_workspace/_autosearch.py +Step 5 (Execute) ──► best_params.json + auto_search.db + quantized_model_*.onnx +``` + +--- + +## Step 1: Model Intake + +Hand off to `quark-onnx-model-intake`. Stop if QDQ nodes already exist — +AutoSearchPro requires an unquantized float `.onnx`. Add one recipe-specific +line to the summary: *"Search will produce ~N quantized .onnx files in +`output_dir` — ensure ≥ N × free."* + +### >>> CHECKPOINT 1: Confirm model analysis + +--- + +## Step 2: Preset / Custom Search-Space Selection + +Pick a built-in preset or accept a user-supplied `search_space` dict. See +[`presets-reference.md`](presets-reference.md) for the full preset table and +deployment-target recommendations. + +For custom search spaces, run `validate_search_space(...)` in your head +against the spec rules in the reference. Compute `discrete_space_size` and +warn if `n_trials` would be clamped. + +### Decision table to show the user + +| Decision | Default | Notes | +|----------|---------|-------| +| `search_space` source | preset name | `XINT8_SEARCH` for NPU CNN, `ADVANCED_SEARCH` for accuracy, etc. | +| `search_algo` | `TPE` | also: `Random`, `CmaEs`, `GPS`, `NSGAII`, `QMC`, `Grid` | +| `n_trials` | `20` | preset default; ~4 calib configs + 20 FastFT with `two_stage_search` | +| `n_jobs` | `1` | parallel processes only when GPU memory allows | +| `two_stage_search` | `True` | grid over calib first, then TPE over FastFinetune | +| `search_metric` | `L2` | or `L1` / `cos` / `psnr` / `ssim`, or set `search_evaluator` | +| `direction` | `minimize` | matches L2 distance | +| `output_dir` | `./autosearch_output/` | holds trial `.onnx`, study DB, log | +| `temp_dir` | `./autosearch_temp/` | float-model dumps for built-in evaluator | +| `study_storage_db` | `auto_search.db` | resumable Optuna study | +| `load_study_if_exists` | `True` | resume from interrupted runs | +| `plot_results` | `False` | `True` → `opt_history.html` + `param_importance.html` | +| device override | `cuda:0 → cpu` if no GPU | see device-gating snippet in reference | + +### >>> CHECKPOINT 2: User confirms preset, `n_trials`, devices, metric + +--- + +## Step 3: Calibration + Evaluation Reader + +Reader patterns mirror `quark-onnx-ptq` Step 3 (vision: +`ImageDataReader`; LLM: tokenizer-driven). AutoSearchPro wraps the reader +with `CachedDataReader` — one pass over data is enough. For evaluator mode +(built-in metric vs. custom `search_evaluator(onnx_path) -> float`) and +`direction` semantics, see [`presets-reference.md`](presets-reference.md). +Pass a distinct `eval_data_reader` to score on a different slice; omit it +to reuse calibration (AutoSearchPro warns). + +### >>> CHECKPOINT 3: Confirm reader + evaluator mode + +--- + +## Step 4: Manifest Generation + +Translate the confirmed plan into a runnable script in the user's workspace +and a `run_manifest.yaml`. Do not write into the Quark repo. + +### Actions + +1. **Script path.** Choose `./_autosearch.py` in the user's + working directory. Never under `examples/` or `tutorials/`. +2. **Pre-check optuna.** If `import optuna` fails in the user env, surface + `pip install optuna` before writing the script. +3. **Build the script body.** Pieces: + + ```python + # user_workspace/_autosearch.py + from quark.onnx import AutoSearchPro + from quark.onnx.quantization.auto_search import get_auto_search_config + # from onnxruntime.quantization.calibrate import CalibrationDataReader + # ... ImageDataReader / tokenizer-driven reader from Step 3 ... + + cfg = get_auto_search_config("XINT8_SEARCH") + cfg["model_input"] = "./models/yolov8n.onnx" + cfg["calib_data_reader"] = ImageDataReader("./calib_data", cfg["model_input"]) + cfg["eval_data_reader"] = None + cfg["output_dir"] = "./autosearch_output" + cfg["temp_dir"] = "./autosearch_temp" + cfg["n_trials"] = 20 + cfg["n_jobs"] = 1 + cfg["search_algo"] = "TPE" + cfg["two_stage_search"] = True + cfg["search_metric"] = "L2" + cfg["direction"] = "minimize" + cfg["study_storage_db"] = "auto_search.db" + cfg["load_study_if_exists"] = True + cfg["plot_results"] = True + + # Apply device override here if env has no CUDA/ROCm — see presets-reference.md + + if __name__ == "__main__": + best = AutoSearchPro(cfg).run() + print("Best params:", best) + ``` + + If the user picked a custom search space, replace `cfg["search_space"]` + instead of (or in addition to) using a preset. + +4. **Write the script to disk** at the chosen path. Print the full body back + to the user — never just say "generated a script". +5. **Build the command:** + + ```bash + python3 ./yolov8n_autosearch.py 2>&1 | tee ./autosearch_output/run.stdout.log + ``` + +6. **Expected output layout:** + + ```text + ./_autosearch.py (generated here) + ./run_manifest.yaml (this recipe's manifest) + ./autosearch_temp/ (float-model dumps; cleaned at end) + ./autosearch_output/ + ├── auto_search.log + ├── auto_search.db (resumable Optuna study) + ├── best_params.json (final answer) + ├── quantized_model_*.onnx (one per trial) + └── opt_history.html / param_importance.html (if plot_results=True) + ``` + +7. **Estimate cost.** From `model_analysis.json`: disk ≈ model_size × + `n_trials`; flag if > 100 GB. Estimate per-trial wall-clock from + CPU-vs-GPU + FastFinetune iter range so the user knows whether to walk + away. + +### >>> CHECKPOINT 4: Show script body, command, cost estimate; ask "shall I run this?" + +If the user declines or wants changes, return to Step 2 (preset/budget) or +Step 3 (reader/evaluator) — never patch the script in place. + +--- + +## Step 5: Execute AutoSearch + +Runs ONLY after explicit Step 4 confirmation. + +### Actions + +1. `mkdir -p ./autosearch_output ./autosearch_temp` +2. Run the script and stream the log. AutoSearchPro logs per-trial: + + ```text + [Trial 7] Params: {'activation': 'Int8Spec', 'algorithms': 'adaround', ...} + L2 distance is: 0.00012 + ``` + +3. After completion, verify: + + ```bash + ls -lh ./autosearch_output/best_params.json ./autosearch_output/auto_search.db + ``` + +4. Read `best_params.json` and report a summary: trials run / budget, best + trial index, best metric value, the resolved params dict, the winning + `quantized_model_*.onnx` path, the study DB path (resumable), and the + plot HTMLs if generated. +5. **Follow-ups:** hand off the winning `.onnx` to + `quark-onnx-result-validator`; prune unwanted trial `.onnx` files if disk + is tight; offer a continuation run with a tighter `search_space` around + the winning region. + +### Common failures and routing + +- `ModuleNotFoundError: optuna` → `pip install optuna`, restart. +- `CUDAExecutionProvider not available` or custom-op library load failure + (`BFPQuantizeDequantize`, `MXQuantizeDequantize`, `Extended*`) → + `quark-onnx-install` / `quark-onnx-debug`; never silently swap to CPU. +- OOM during AdaRound/AdaQuant → narrow `num_iterations` range, lower + `data_size`, drop `batch_size`, or set `optim_device = ["cpu"]`. +- Resumed run loaded an incompatible study → delete `auto_search.db` or pick + a new `study_name`. +- "n_trials clamped to discrete_space_size" → expected when search space has + no continuous fields and is exhausted; informational. + +## Recovery + +- If any upstream artifact is missing, stop and name the producer skill. +- If the model is already QDQ-quantized, stop — AutoSearchPro requires + unquantized input. +- If the user wants to change preset / budget mid-run, return to Step 2 — + do not patch the generated script. +- If `optuna` cannot be installed, surface a clear blocker; there is no + fallback search backend. +- If the user pastes a Torch traceback or HuggingFace path, hand off to + `the matching public quark-torch skill` and stop. + +## See also + +- [`presets-reference.md`](presets-reference.md) — presets, custom-space + rules, device gating, evaluator modes, samplers. +- `quark-onnx-ptq` — single-shot PTQ alternative. +- `quark-onnx-result-validator`, `quark-onnx-debug`, `quark-onnx-install`. diff --git a/skills/quark-onnx-autosearch-pro/presets-reference.md b/skills/quark-onnx-autosearch-pro/presets-reference.md new file mode 100644 index 0000000..d2d9e9d --- /dev/null +++ b/skills/quark-onnx-autosearch-pro/presets-reference.md @@ -0,0 +1,72 @@ +# AutoSearchPro Preset Reference + +Source: `quark/onnx/quantization/auto_search/qconfig_mapping.py`. Fetch via +`get_auto_search_config(name)`. + +## Built-in presets + +| Preset | Activation | Weight | Calibration | Algorithms | Best for | +|--------|-----------|--------|-------------|-----------|----------| +| `ADVANCED_SEARCH` | Int8 sym+asym | Int8 sym | MinMax / Percentile / LayerwisePercentile | adaround, adaquant (LR 1e-6→1e-3, iters 3k→30k) | Accuracy-first, slowest | +| `XINT8_SEARCH` | XInt8 sym | XInt8 sym tensor | MinMSE | cle + adaround/adaquant (iters 3k→20k) | AMD Ryzen AI NPU CNN | +| `A8W8_SEARCH` | Int8 sym+asym | Int8 sym tensor | MinMax / Percentile / LayerwisePercentile + `PercentileCandidates` | adaround, adaquant, cle, DPU alignments | Generic CPU/GPU INT8 | +| `A16W8_SEARCH` | Int16 sym+asym | Int8 sym tensor | MinMax / Percentile | cle + adaround/adaquant, `AlignEltwiseQuantType` | Accuracy-sensitive (W8 lost too much) | + +All defaults: `search_algo = "TPE"`, `direction = "minimize"`, `n_trials = 20`, +`n_jobs = 1`, `two_stage_search = True`, `optim_device = "cuda:0"`, +`study_storage_db = "auto_search.db"`, `load_study_if_exists = True`. + +## Recommended preset by deployment + +| Deployment | Priority | Recommended preset | +|------------|----------|--------------------| +| AMD NPU CNN (Ryzen AI) | Best accuracy | `XINT8_SEARCH` | +| AMD NPU Transformer | Best accuracy | custom: `BFP16Spec` weights + Int8 act | +| CPU / CUDA / ROCm general | Best accuracy | `ADVANCED_SEARCH` | +| CPU / CUDA / ROCm general | Smallest model | `A8W8_SEARCH` | +| Any | Recover lost accuracy from W8A8 | `A16W8_SEARCH` | +| Any | Custom search space provided | use as-is, validate first | + +## Custom search-space rules + +`quark.onnx.quantization.auto_search.utils.validate_search_space` enforces: + +- **Base fields** → `list[T]`. +- **Conditional fields** → `dict` containing `"only_if"` (string ⇒ that field + must be set; `{field: value}` ⇒ exact match) plus list-valued parameters. +- **Continuous parameters** → `{"type": "int" | "float", "low": …, "high": …, + "step": …, "log": …}`. + +Returns `{"discrete_space_size": int, "contains_continuous": bool}`. If +`n_trials > discrete_space_size` and no continuous fields, AutoSearchPro +auto-clamps — surface this instead of letting it happen silently. + +## Device gating + +For every list-valued device field in the selected `search_space` +(`adaround_params.{optim,infer}_device`, `adaquant_params.{optim,infer}_device`), +replace `"cuda:0"` with `"cpu"` when `env_context.json` reports no CUDA/ROCm: + +```python +for key in ("adaround_params", "adaquant_params"): + if key in cfg["search_space"]: + cfg["search_space"][key]["optim_device"] = ["cpu"] + cfg["search_space"][key]["infer_device"] = ["cpu"] +``` + +Never silently keep the CUDA value. + +## Evaluator modes + +| Mode | What you set | Returns | When to use | +|------|--------------|---------|-------------| +| Built-in metric | `search_metric = "L2" \| "L1" \| "cos" \| "psnr" \| "ssim"`; `search_evaluator = None` | Per-sample distance between float and quantized output `.npy` dumps | Default — quick, no extra code | +| Custom evaluator | `search_evaluator = fn` where `fn(onnx_path) -> float` | Whatever your task metric returns; AutoSearchPro uses `base_metric - quantized_metric` | Detection mAP, top-1, perplexity, BLEU | + +Keep `direction = "minimize"` and return `−metric` from `fn` when your metric +is higher-is-better. + +## Sampler choices + +`search_algo` accepts: `TPE` (default), `Random`, `CmaEs`, `GPS`, `NSGAII`, +`QMC`, `Grid`. See `qconfig_mapping.get_sampler_dict`. diff --git a/skills/quark-onnx-autosearch-pro/skill-card.md b/skills/quark-onnx-autosearch-pro/skill-card.md new file mode 100644 index 0000000..3f988d8 --- /dev/null +++ b/skills/quark-onnx-autosearch-pro/skill-card.md @@ -0,0 +1,13 @@ +# Skill Card + +## Description + +End-to-end Quark ONNX AutoSearchPro recipe — drives `quark.onnx.AutoSearchPro` (Optuna-based hyperparameter search) on a `.onnx` model to find the best quantization config (activation/weight spec, calibration method, CLE, AdaRound / AdaQuant, FastFinetune params). + +## Owner + +amd (federated from [amd/Quark](https://github.com/amd/Quark)) + +## License + +MIT diff --git a/skills/quark-onnx-debug/.federated.json b/skills/quark-onnx-debug/.federated.json new file mode 100644 index 0000000..771c78e --- /dev/null +++ b/skills/quark-onnx-debug/.federated.json @@ -0,0 +1,10 @@ +{ + "source": "amd-quark", + "repo": "amd/Quark", + "ref": "v0.12", + "commit": "1b229f781a1974cc742884e42d8eefc1eebb4f0a", + "path": ".claude/skills/quark-onnx-debug", + "license": "MIT", + "imported_at": "2026-08-05T21:07:39Z", + "implementation_path": ".claude/skills-impl/l1-atomic/onnx/quark-onnx-debug" +} diff --git a/skills/quark-onnx-debug/SKILL.md b/skills/quark-onnx-debug/SKILL.md new file mode 100644 index 0000000..aa6b487 --- /dev/null +++ b/skills/quark-onnx-debug/SKILL.md @@ -0,0 +1,280 @@ +--- +name: quark-onnx-debug +description: >- + Diagnose failed Quark ONNX installation, calibration, quantization, custom-op compilation, or + export attempts. Use when the user reports an error, stack trace, invalid artifact, missing + dependency, ORT execution-provider mismatch, silent CPU fallback, OOM during calibration, + custom-op load failure (BFPQuantizeDequantize / MXQuantizeDequantize / Extended*), or unexpected + quantization results from the ONNX flow. Trigger for "Quark ONNX error", "onnxruntime error", + "quantize_static failed", "calibration crashed", "CUDAExecutionProvider not available", + "ROCMExecutionProvider not available", "custom op library load failed", "model.onnx larger than + 2GB", "external data not found", "AdaRound diverged", "GPTQ ONNX failed", "QuaRot failed", "NPU + power-of-2 scale", any Python traceback mentioning quark.onnx / onnxruntime / onnx. +--- + +# quark-onnx-debug + +## Catalog portability + +This standalone skill is federated from `amd/Quark` at commit `1b229f781a1974cc742884e42d8eefc1eebb4f0a`. Resolve bundled files relative to this `SKILL.md`. Repository-relative paths such as `docs/`, `examples/`, and `quark/` refer to the [pinned Quark source](https://github.com/amd/Quark/tree/1b229f781a1974cc742884e42d8eefc1eebb4f0a); use a local Quark checkout when available, otherwise consult that pinned source. + +## Purpose + +Convert ONNX-flow failures into a structured diagnostic report with the smallest safe recovery path. +Debugging Quark ONNX issues is tricky because errors can originate from many layers — Python +environment, the `onnx` package version, the installed `onnxruntime*` variant, custom-op +compilation, CUDA / ROCm / NPU EPs, calibration data plumbing, graph optimization, or the chosen +quantization config itself. This skill systematically narrows down the root cause. + +## Inputs + +- Error message and stack trace from a failing run +- The exact command or `ModelQuantizer` / `quantize_static` invocation that triggered it +- `env_context.json`, `onnx_install_result.json`, `quark_install_result.json` (optional, for + environment and install state) +- The model path / size and (if relevant) the `QuantizationConfig` used + +## Outputs: validation_report.md + +Diagnostic report with root cause, evidence, and the smallest safe fix. + +Schema: [`validation_report.schema.json`](https://github.com/amd/Quark/blob/1b229f781a1974cc742884e42d8eefc1eebb4f0a/.claude/skills-impl/shared/contracts/validation_report.schema.json) + +````markdown +# Debug Report + +## Common Error Patterns + +### Installation Errors + +| Error | Likely Cause | Fix | +|-------|-------------|-----| +| `ModuleNotFoundError: No module named 'quark'` | Quark not installed or wrong Python env | `pip install amd-quark` or activate correct conda env | +| `ModuleNotFoundError: No module named 'onnxruntime'` | ONNX Runtime not installed | Hand off to `quark-onnx-install` | +| `ModuleNotFoundError: No module named 'onnx'` | `onnx` package not installed | `pip install "onnx>=1.21.0,<=1.22.0"` | +| `ImportError: quark.onnx.operators.custom_ops` / compile failure | Missing C++ compiler (Linux `g++`, Windows VS 2022) or `ROCM_PATH`/`CUDA_HOME` unset for GPU build | `apt install g++` (Linux), install VS 2022 (Windows), `export ROCM_PATH=/opt/rocm` or `export CUDA_HOME=/usr/local/cuda` | +| Both `onnxruntime` and `onnxruntime-gpu` (or `_rocm`) installed | Variant collision — wrong EP loads | `pip uninstall -y onnxruntime onnxruntime-gpu onnxruntime_rocm`, then reinstall the single correct variant via `quark-onnx-install` | +| `onnx` schema / `opset` errors at import | `onnx` outside `>=1.16.0,<=1.19.0` | Pin within the supported range | +| `onnxruntime` ABI / symbol errors when loading custom ops | ORT version outside `>=1.22.2,<=1.24.2` (custom-ops built against a different ABI) | Reinstall ORT within the supported range, then re-trigger custom-ops compile | + +### ONNX Runtime / Execution Provider Errors + +| Error | Likely Cause | Fix | +|-------|-------------|-----| +| `get_available_providers()` lacks `CUDAExecutionProvider` | CPU `onnxruntime` installed on a CUDA box | Reinstall `onnxruntime-gpu` via `quark-onnx-install` | +| `get_available_providers()` lacks `ROCMExecutionProvider` on ROCm 6.x | Wrong variant (need `onnxruntime_rocm` from AMD Artifactory) | Reinstall `onnxruntime_rocm` via `quark-onnx-install` | +| `ROCMExecutionProvider` missing on ROCm 7.x | **By design** — ROCm 7.x falls back to CPU `onnxruntime` per `tools/ci/install_onnxruntime.sh` (build incompatibility) | Document the CPU-only fallback; do not attempt a ROCm wheel install | +| Silent CPU fallback (no GPU utilization during calibration) | EP not requested or unavailable | Pass `execution_providers=['CUDAExecutionProvider']` / `['ROCMExecutionProvider']` explicitly; verify with `get_available_providers()` | +| `Failed to load library libonnxruntime_providers_cuda.so` | CUDA toolkit version mismatch with `onnxruntime-gpu` build | Match `onnxruntime-gpu` to the system CUDA major (CUDA 11 → Azure DevOps index; CUDA 12/13 → pypi default) | +| `RuntimeError: ... onnxruntime::rocm::...` | ROCm driver / library mismatch | Verify `rocm-smi` matches the wheel's ROCm version | + +### Custom Op Errors + +| Error | Likely Cause | Fix | +|-------|-------------|-----| +| `RuntimeError: Failed to load custom op library` | Custom-ops library was not compiled, or compiled against a different ORT version | Re-run `python -c "import quark.onnx.operators.custom_ops"` and inspect the compile output | +| `Op (BFPQuantizeDequantize) ... is not a registered function/op` | The session was created without registering Quark's custom-op library | Use `quark.onnx.ModelQuantizer` / `quantize_static`, or pass the custom-op `.so`/`.dll` via `SessionOptions.register_custom_ops_library()` | +| Custom-op symbol-not-found on GPU but works on CPU | GPU kernel of the custom op was not built (missing `ROCM_PATH` / `CUDA_HOME` at compile time) | Set the env var, delete the cached `.so`/`.dll`, re-import to recompile | + +### Model Loading / Shape Errors + +| Error | Likely Cause | Fix | +|-------|-------------|-----| +| `FileNotFoundError: Input model file ... does not exist.` (`api.py:103`) | Wrong `model_input` path | Check absolute path to `model.onnx` | +| `onnx.onnx_cpp2py_export.checker.ValidationError` | Model fails ONNX schema check | Run `onnx.checker.check_model()` to localize; rebuild the model with a compatible opset | +| `Message ... exceeds 2GB` / ProtoBuf size error | Model > 2 GB without external data | Save with `save_as_external_data=True`; pass `use_external_data_format=True` to the quantizer | +| `external data file not found` | `.onnx` moved but `.onnx_data` / weight blobs left behind | Move the model directory as a whole, or re-export with external data adjacent | +| Shape-inference failure | Incomplete shapes in the model | Run `quark.onnx.tools.fix_shapes` / `onnx.shape_inference.infer_shapes` before quantization | + +### Calibration Errors + +| Error | Likely Cause | Fix | +|-------|-------------|-----| +| `ValueError: The data reader should implement the '__len__' method to provide the data size.` (`calibrators.py:239`) | Custom `CalibrationDataReader` missing `__len__` | Implement `__len__` returning sample count | +| `ValueError: No data is collected.` (`calibrators.py:253`) | Data reader yielded zero batches before exhaustion | Check reader's `get_next()` returns at least one batch | +| `TypeError: compute_data must return a TensorsData not <...>` (`calibrators.py:259`) | Custom calibrator subclass returned wrong type | Return `onnxruntime.quantization.calibrate.TensorsData` | +| `ValueError: No collector created and can't generate calibration data.` | `collect_data()` never called before `compute_data()` | Call `collect_data()` first, or use `ModelQuantizer.quantize_model()` which handles the order | +| `ValueError: Invalid averaging constant, which should not be < 0 or > 1.` (`calibrators.py:563`) | `moving_average=True` with out-of-range constant | Use a value in `[0, 1]`, typically `0.01` | +| `ValueError: Unsupported calibration method` (`calibrators.py:1237`) | Typo or unsupported `CalibrationMethod` | Check `CalibrationMethod` enum in `quark.onnx.quantization.config.config` | +| OOM during calibration | Activation cache too large | Set `optimize_mem=True` (disk cache) and/or `optimize_disk=True`; reduce `worker_num` if memory bound; reduce calibration sample count | + +### Quantization Config Errors + +| Error | Likely Cause | Fix | +|-------|-------------|-----| +| `ValueError: Only ExtendedQuantFormat.QDQ supports wide bits quantization types.` (`input_check.py:67`) | INT16/UINT16/INT32 with `QuantFormat.QOperator` | Switch to `ExtendedQuantFormat.QDQ` | +| `ValueError: Fast finetune does not support int4 or uint4.` (`input_check.py:118`) | AdaRound/AdaQuant + INT4 weights | Disable fast-finetune for INT4, or use GPTQ instead | +| `ValueError: Invalid quant overrides ... for tensor ...` (`input_check.py:86`) | Tensor name in `extra_options['QuantOverrides']` not in the graph | Verify tensor name with `onnx.load(...).graph` | +| `ValueError: The per-channel quant override ... can not be applied on ` (`input_check.py:94`) | Per-channel override on a quant type that doesn't support it (e.g., FP types, block formats) | Remove per-channel from that override | +| `ValueError: For the crypto mode, the input model should be in onnx.ModelProto format.` (`input_check.py:137`) | `crypto_mode=True` with a file path | Load model first: `model = onnx.load(path)` | +| `ValueError: quantization config must be one of Config and QConfig.` (`api.py:73`) | Passed a dict or wrong type | Build a `quark.onnx.quantization.config.Config` or `QConfig` | +| `ValueError: Unexpected config name: ` (`custom_config.py:993`) | Typo in `get_default_config()` preset name | Pick from `DefaultConfigMapping` (XINT8, S8S8_AAWS, A8W8, A16W8, BF16, BFP16, MX4/6/9, MXFP8/6/4, …) | +| `nodes_to_quantize` ignored | Node names don't match graph after pre-processing (NCHW→NHWC, BN folding) | Run pre-processing first, dump the post-preprocess graph, then re-select node names | + +### Algorithm Errors + +| Error | Likely Cause | Fix | +|-------|-------------|-----| +| AdaRound / AdaQuant device mismatch | `optim_device='cuda:0'` but no `CUDAExecutionProvider` available | Either set `optim_device='cpu'` or fix the ORT install via `quark-onnx-install` | +| AdaRound / AdaQuant divergence (loss → NaN/Inf) | LR too high for the model, or activation outliers | Lower `learning_rate`, enable `CLE` pre-processing first | +| GPTQ shape mismatch | Group-size doesn't divide the weight dimension | Pick a `group_size` that divides the weight's reduction dim (commonly 32, 64, 128) | +| SmoothQuant alpha range | `SmoothAlpha` outside `[0, 1]` | Use a value in `[0.5, 0.85]` for LLMs | +| QuaRot rotation config invalid | Missing rotation pair definitions | Provide the rotation pairs in the algorithm config; see `quark/onnx/algorithm/quarot/` | +| `AutoSearchPro: 'model_input' can not be None.` (`auto_search_pro.py:139`) | Search invoked without a model | Pass the model path or `ModelProto` | +| `AutoSearchPro: 'calib_data_reader' can not be None.` (`auto_search_pro.py:142`) | Search invoked without calibration data | Provide a `CalibrationDataReader` | +| `AutoSearchPro: Unsupported search_algo` (`auto_search_pro.py:192`) | Sampler typo | Use `'TPE'` or `'Grid'` | + +### NPU-Specific Errors (Ryzen AI / VAI) + +| Error | Likely Cause | Fix | +|-------|-------------|-----| +| Power-of-2 scale validation failure | Non-PoF2 scale produced for an NPU target | Use a PoF2 calibrator (`PowOfTwoCalibrater` MinMSE / NonOverflow); confirm `enable_npu_cnn=True` | +| `enable_npu_cnn=True` but model has unsupported ops | NPU CNN backend doesn't support the op | Fold / replace via `quark.onnx.tools.*`; see `optimizations/optimize.py` | +| NCHW vs NHWC mismatch | NPU expects NHWC but model is NCHW | Run `quark.onnx.tools.convert_nchw_to_nhwc` before quantization | +| `enable_dpu` deprecated warning | Old API | Switch to `enable_npu_cnn=True` | + +### Export / Post-quantization Errors + +| Error | Likely Cause | Fix | +|-------|-------------|-----| +| `PermissionError` on output dir | No write permission | Check output directory permissions | +| Quantized model > 2 GB and external data not written | `use_external_data_format=False` on a large model | Set `use_external_data_format=True` | +| Inference using the quantized model errors on a custom op | Deployment env missing Quark's custom-ops library | Ship the compiled `.so`/`.dll` and register via `SessionOptions.register_custom_ops_library()` | + +## Diagnostic Process + +1. **Read the error** — capture the exact error message, full stack trace, and the command or + `quantize_static` / `ModelQuantizer` call that triggered it. +2. **Identify the layer** — installation / ORT runtime / custom-op / model loading / calibration / + config validation / algorithm / NPU-specific / export. +3. **Check the environment** — Python version, `onnx` version, installed `onnxruntime*` variants + and versions, available EPs, Quark version, accelerator (CUDA major / ROCm major), whether the + custom-ops library compiled. +4. **Match against known patterns** — use the tables above. +5. **Propose the fix** — smallest change that resolves the issue without side effects. + +## Diagnostic Commands + +```bash +# Environment snapshot +python -c " +import sys; print('Python:', sys.version.split()[0]) +try: + import onnx; print('onnx:', onnx.__version__) +except Exception as e: print('onnx: not installed', e) +try: + import onnxruntime as ort + print('onnxruntime:', ort.__version__) + print('EPs:', ort.get_available_providers()) +except Exception as e: print('onnxruntime: not installed', e) +try: + import quark; print('Quark:', quark.__version__) +except Exception as e: print('Quark: not installed', e) +try: + import quark.onnx; print('quark.onnx loaded from', quark.onnx.__file__) +except Exception as e: print('quark.onnx: load failed', e) +" + +# Confirm only ONE onnxruntime variant is installed +pip list 2>/dev/null | grep -iE '^(onnx|onnxruntime|onnxslim|onnxscript|onnxruntime-genai|onnxruntime_rocm|onnxruntime-extensions)\b' + +# Force custom-ops compile (first import) and capture failures +python -c "import quark.onnx.operators.custom_ops" 2>&1 | tail -30 + +# Model sanity check +python -c " +import onnx, sys +m = onnx.load(sys.argv[1]) +print('opset:', [(o.domain or 'ai.onnx', o.version) for o in m.opset_import]) +print('inputs:', [(i.name, [d.dim_value or d.dim_param for d in i.type.tensor_type.shape.dim]) for i in m.graph.input]) +onnx.checker.check_model(m, full_check=False) +print('checker: OK') +" path/to/model.onnx + +# GPU memory status +nvidia-smi --query-gpu=memory.used,memory.total --format=csv,noheader 2>/dev/null +rocm-smi --showmemuse 2>/dev/null +``` + +## Rules + +- **Always ask for the full error message and the call that was run.** Partial errors lead to wrong + diagnoses. For ONNX, also ask for: the `QuantizationConfig` / preset name, the model path and + size, the chosen `execution_providers`, and the calibration data reader class. +- **Do not guess the fix.** Narrow down the root cause first, then propose a specific solution. +- **Distinguish "EP not available" from "EP available but not requested".** Both cause silent CPU + fallback. Check `ort.get_available_providers()` *and* the providers actually passed to the + session. +- **Confirm before mutating.** If the fix involves reinstalling `onnxruntime*`, rebuilding the + custom-ops library, or re-running an hour-long calibration, present the plan and get + confirmation. +- **Consider cascading effects.** Bumping `onnxruntime` may force a custom-ops rebuild and may break + ABI with an older `onnx` pin. Bumping `onnx` outside `<=1.19.0` may break Quark's QDQ insertion. + Always check version constraints (`requirements.txt`, `docs/source/install.rst`). +- **For ROCm 7.x with no ROCM EP, this is by design.** Do not propose a "fix" that downgrades to + ROCm 6.x unless the user asks; instead document the CPU-fallback trade-off (see + `tools/ci/install_onnxruntime.sh`). + +## Example: CUDA EP missing during calibration + +### Error Summary + +`onnxruntime.capi.onnxruntime_pybind11_state.RuntimeException: ... CUDAExecutionProvider is not in +the list of available providers` raised from `ModelQuantizer.quantize_model()` on a CUDA 12 box. + +### Root Cause + +CPU `onnxruntime` was installed instead of `onnxruntime-gpu`; `get_available_providers()` returns +`['CPUExecutionProvider']` only. + +### Evidence + +- `pip list | grep onnxruntime` shows `onnxruntime 1.23.2` only (no `-gpu` variant). +- `python -c "import onnxruntime as ort; print(ort.get_available_providers())"` → + `['CPUExecutionProvider']`. +- `nvidia-smi` reports a healthy CUDA 12.4 driver and an idle GPU. + +### Fix + +Reinstall the correct variant via `quark-onnx-install` (CUDA 12 path): + +```bash +pip uninstall -y onnxruntime onnxruntime-gpu onnxruntime_rocm +pip install --no-cache-dir onnxruntime-gpu +python -c "import onnxruntime as ort; print(ort.get_available_providers())" # expect CUDAExecutionProvider present +python -c "import quark.onnx.operators.custom_ops" # recompile against new ORT +``` + +### Prevention + +Run `quark-onnx-install` (which routes through `quark-env-preflight`) before the first +quantization on a fresh environment, so the ORT variant is picked from the accelerator instead of +defaulting to CPU. +```` + +## Interaction Flow + +1. **Gather evidence**: error message, stack trace, exact call, model size, config, EPs. +2. **Classify**: install / custom-op / ORT runtime / model / calibration / config / algorithm / + NPU / export. +3. **Diagnose**: match against known patterns and run diagnostic commands if needed. +4. **Propose fix**: present the smallest change that resolves the issue. +5. **Confirm**: get user approval before reinstalls, custom-ops rebuilds, or long recalibrations. +6. **Verify**: after the fix, re-run the failing step (or a smaller smoke variant) to confirm + resolution. + +## Recovery + +- If the fix requires an `onnxruntime*` or `onnx` package change (wrong variant, version + mismatch, missing variant), hand off to `quark-onnx-install` with the specific requirement noted. +- If the fix requires an `amd-quark` package or shared-dependency change, hand off to + `quark-install` with the specific requirement noted. +- If the fix requires a C++ compiler install or `ROCM_PATH`/`CUDA_HOME` setup, surface the + environment gap and let the user resolve it before re-running custom-ops compile. +- If the root cause is upstream (ORT API change, `onnx` schema change), hand off to + `the Quark repository's internal ONNX documentation drift checker` or `the Quark repository's internal ONNX skill synchronizer`. +- If the error is in a custom config or unsupported op for an NPU target, suggest the relevant + pre-processing tool from `quark.onnx.tools` (e.g. `convert_nchw_to_nhwc`, + `convert_qdq_to_qop`, `convert_a8w8_npu_to_a8w8_cpu`). +- If the issue cannot be reproduced from the supplied evidence, ask for the minimum repro + (model, calibration data sample, config) before guessing. diff --git a/skills/quark-onnx-debug/skill-card.md b/skills/quark-onnx-debug/skill-card.md new file mode 100644 index 0000000..f9b2c88 --- /dev/null +++ b/skills/quark-onnx-debug/skill-card.md @@ -0,0 +1,13 @@ +# Skill Card + +## Description + +Diagnose failed Quark ONNX installation, calibration, quantization, custom-op compilation, or export attempts. Use when the user reports an error, stack trace, invalid artifact, missing dependency, ORT execution-provider mismatch, silent CPU fallback, OOM during calibration, custom-op load failure (BFPQuantizeDequanti… + +## Owner + +amd (federated from [amd/Quark](https://github.com/amd/Quark)) + +## License + +MIT diff --git a/skills/quark-onnx-install/.federated.json b/skills/quark-onnx-install/.federated.json new file mode 100644 index 0000000..a964b24 --- /dev/null +++ b/skills/quark-onnx-install/.federated.json @@ -0,0 +1,10 @@ +{ + "source": "amd-quark", + "repo": "amd/Quark", + "ref": "v0.12", + "commit": "1b229f781a1974cc742884e42d8eefc1eebb4f0a", + "path": ".claude/skills/quark-onnx-install", + "license": "MIT", + "imported_at": "2026-08-05T21:07:39Z", + "implementation_path": ".claude/skills-impl/l1-atomic/onnx/quark-onnx-install" +} diff --git a/skills/quark-onnx-install/SKILL.md b/skills/quark-onnx-install/SKILL.md new file mode 100644 index 0000000..e8014aa --- /dev/null +++ b/skills/quark-onnx-install/SKILL.md @@ -0,0 +1,277 @@ +--- +name: quark-onnx-install +description: >- + Install or verify the correct ONNX Runtime build (and the matching `onnx` package) for a user's + accelerator backend before Quark ONNX-flow usage. Trigger for "install onnxruntime", "pip + install onnxruntime", "set up onnxruntime for ROCm", "set up onnxruntime for CUDA", + "onnxruntime-gpu vs onnxruntime", "onnx version mismatch", "CPU-only onnxruntime installed", + "onnxruntime providers list missing CUDAExecutionProvider/ROCMExecutionProvider", failures + importing `onnxruntime`, or any request to get the correct ONNX Runtime build running. Also + trigger when quark-install reports that ONNX Runtime is missing or mismatched before proceeding + with the ONNX-to-ONNX flow. +--- + +# quark-onnx-install + +## Catalog portability + +This standalone skill is federated from `amd/Quark` at commit `1b229f781a1974cc742884e42d8eefc1eebb4f0a`. Resolve bundled files relative to this `SKILL.md`. Repository-relative paths such as `docs/`, `examples/`, and `quark/` refer to the [pinned Quark source](https://github.com/amd/Quark/tree/1b229f781a1974cc742884e42d8eefc1eebb4f0a); use a local Quark checkout when available, otherwise consult that pinned source. + +## Purpose + +Install the correct ONNX Runtime build for the user's accelerator backend, plus the matching `onnx` +package and supporting tooling (`onnxslim`, `onnxscript`). ONNX Runtime must be installed before +Quark's ONNX-to-ONNX flow because Quark uses ORT as the calibration / inference engine and registers +custom ops (`BFPQuantizeDequantize`, `MXQuantizeDequantize`, `Extended*`) into it. Getting this wrong — +installing the CPU build on a GPU machine, or installing both `onnxruntime` and `onnxruntime-gpu` +side-by-side — causes EP-not-available errors, silent CPU fallback, or import-time DLL conflicts that +are hard to diagnose later. This skill exists separately from `quark-install` so that ONNX Runtime +setup has a clear, single-responsibility boundary, parallel to `quark-torch-install` for Torch. + +## Inputs + +- `env_context.json` with detected accelerator info (CPU / CUDA major+minor / ROCm major+minor) + +## Outputs: onnx_install_result.json + +Records the installed ONNX Runtime build, accelerator backend tag, the `onnx` package version, and +verification status. + +```json +{ + "status": "ok", + "onnxruntime_package": "onnxruntime-gpu", + "onnxruntime_version": "1.23.2", + "accelerator_tag": "cuda-12", + "onnx_version": "1.18.0", + "onnxslim_version": "0.1.84", + "onnxscript_version": "0.1.0", + "verification": { + "import_onnx_ok": true, + "import_onnxruntime_ok": true, + "available_providers": ["CUDAExecutionProvider", "CPUExecutionProvider"], + "expected_provider_present": true, + "custom_ops_compile_ok": true + } +} +``` + +On failure, set `status: "failed"` and include a `failure_reason` with the exact failing verification +command. + +## Python Version Requirements + +- **Supported**: Python 3.11, 3.12, 3.13 +- **Not supported**: Python 3.14+ +- **Recommended for new setups**: Python 3.13 via Miniforge/Miniconda + +## Package Version Matrix + +**Authoritative sources**: + +- `tools/ci/install_onnxruntime.sh` — accelerator → `onnxruntime*` variant + version mapping (CI truth) +- `docs/source/install.rst` — user-facing supported version range +- `requirements.txt` — `onnx`, `onnxscript`, `onnxslim` pin + +Before generating install commands, **always read these sources** to get the current verified +combinations. Do not memorize version numbers — the matrix changes with each Quark release. + +### Current ranges (verify before use) + +| Package | Range (verify against requirements.txt / install.rst) | +|---------|-------------------------------------------------------| +| `onnx` | `>=1.21.0, <=1.22.0` | +| `onnxruntime*` | `>=1.22.2, <=1.25.1` | +| `onnxslim` | `>=0.1.84` | +| `onnxscript` | unpinned | + +### How to read the source + +1. Open `tools/ci/install_onnxruntime.sh` and locate the `install_onnxruntime` function. It dispatches + on `accelerator_version` (`cpu`, `cuda-11.*`, `cuda-12.*`, `rocm-*`) and decides: + - which variant to install (`onnxruntime`, `onnxruntime-gpu`, `onnxruntime_rocm`), + - whether to use pypi.org or the AMD internal Artifactory wheel. +2. Cross-check the chosen `onnxruntime` version against the range in `docs/source/install.rst` + (search for "ONNX Runtime version"). +3. Read `requirements.txt` for the `onnx` / `onnxslim` / `onnxscript` constraints. +4. Construct the install commands using the patterns below. + +### Install command patterns + +#### CPU + +```bash +pip install "onnxruntime>=1.22.2,<=1.25.1" +pip install "onnx>=1.21.0,<=1.22.0" "onnxslim>=0.1.84" onnxscript +``` + +#### CUDA 12.x / 13.x + +Matches the current `install.rst` recommendation: + +```bash +pip install onnxruntime-gpu # default pypi build targets recent CUDA +pip install "onnx>=1.21.0,<=1.22.0" "onnxslim>=0.1.84" onnxscript +``` + +#### CUDA 11.x + +Per `install_onnxruntime.sh`: + +```bash +pip install --no-cache-dir onnxruntime-gpu \ + --extra-index-url https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/onnxruntime-cuda-11/pypi/simple/ +pip install "onnx>=1.21.0,<=1.22.0" "onnxslim>=0.1.84" onnxscript +``` + +#### ROCm 6.x + +Internal `onnxruntime_rocm` wheel from AMD Artifactory (no pypi build): + +```bash +# Resolved via _install_onnxruntime_from_artifactory in tools/ci/install_onnxruntime.sh +# Wheel pattern: onnxruntime_rocm--cp-*.whl +# Base URL: https://xcoartifactory.xilinx.com/artifactory/uai-pip-local/onnxruntime/rocm- +``` + +If the user does not have access to `xcoartifactory.xilinx.com`, **stop** and surface the gap — do +not silently fall back to a CPU build. + +#### ROCm 7.x and above + +Per `install_onnxruntime.sh`, build incompatibilities mean the CPU variant is used: + +```bash +pip install "onnxruntime>=1.22.2,<=1.25.1" # CPU variant; ROCm EP not available in this case +pip install "onnx>=1.21.0,<=1.22.0" "onnxslim>=0.1.84" onnxscript +``` + +Make this trade-off explicit to the user (no `ROCMExecutionProvider`, calibration runs on CPU). + +#### Optional: ONNX Runtime GenAI (OGA flow for LLM models) + +```bash +pip install onnxruntime-genai +``` + +#### Optional: ONNX Runtime Extensions + +Referenced in `pyproject.toml` mypy config: + +```bash +pip install onnxruntime-extensions +``` + +**Critical**: Never install both `onnxruntime` and `onnxruntime-gpu` (or `onnxruntime_rocm`) into the +same environment — pip allows it but the imports collide and ORT may load the wrong shared library. +If a different variant is already installed, **uninstall it first** (`pip uninstall -y onnxruntime +onnxruntime-gpu onnxruntime_rocm onnxruntime-genai`). + +## C++ Compiler Requirement + +Quark's ONNX custom-ops library (`quark.onnx.operators.custom_ops`, providing `BFPQuantizeDequantize`, +`MXQuantizeDequantize`, `Extended*`) is **compiled on first import** using the local toolchain. This +must succeed for any BFP / MX / Extended quant scheme to work. + +| OS | Required compiler | +|----|-------------------| +| Linux | `g++` (`apt install g++` on Ubuntu) | +| Windows | Visual Studio 2022+ with the *Desktop development with C++* workload (use the Developer Command Prompt) | + +For GPU kernels, set the corresponding env var so the compiler can find headers: + +- ROCm: `export ROCM_PATH=/opt/rocm` +- CUDA: `export CUDA_HOME=/usr/local/cuda` + +Verify the compile by running: + +```bash +python -c "import quark.onnx.operators.custom_ops" +``` + +## Rules + +- **Always read `tools/ci/install_onnxruntime.sh` before generating install commands.** The version + matrix and Artifactory paths change with each Quark release. Never rely on memorized version + numbers — always verify against the upstream script and `requirements.txt`. +- **Always detect the accelerator before choosing the ORT variant.** Run or reference + `quark-env-preflight` if hardware facts are missing. The entire install plan depends on getting + this right (CPU `onnxruntime`, GPU `onnxruntime-gpu`, ROCm 6.x `onnxruntime_rocm`, ROCm 7.x falls + back to CPU `onnxruntime`). +- **Bind ORT variant and accelerator to the same backend.** Never mix `onnxruntime-gpu` (CUDA) with a + ROCm environment or vice versa. If multiple `onnxruntime*` variants are detected installed, + uninstall all of them before installing the correct one. +- **Pin within the supported ranges.** `onnx` must be `>=1.21.0,<=1.22.0` per `requirements.txt`; + ORT must be in the range stated in `docs/source/install.rst`. Versions outside these ranges + silently break Quark's QDQ insertion or custom-op registration. +- **Never skip verification.** After installation, always run the verification commands below, + including the custom-ops compile check. +- **If accelerator or AMD Artifactory access is unclear, stop after the plan.** Present the install + plan but do not execute. Hand the gap back to `the matching public quark-onnx skill` so it lands + in `session_context.json`'s `open_questions`, and ask the user to confirm. +- **Show exact commands before execution.** The user should see every `pip uninstall` / + `pip install` command, every version, and every `--extra-index-url` before anything runs. + +## Verification Commands + +```bash +# onnx package check +python -c "import onnx; print('onnx:', onnx.__version__)" + +# onnxruntime check + EP list +python -c "import onnxruntime as ort; print('ORT:', ort.__version__); print('EPs:', ort.get_available_providers())" + +# Expected EPs (assert at least one of these is in the list): +# CPU: 'CPUExecutionProvider' +# CUDA: 'CUDAExecutionProvider' (and 'CPUExecutionProvider') +# ROCm 6.x: 'ROCMExecutionProvider' (and 'CPUExecutionProvider') +# ROCm 7.x: 'CPUExecutionProvider' only (no ROCm EP — by design, see install_onnxruntime.sh) + +# Quark ONNX custom-ops compile (first run triggers compilation) +python -c "import quark.onnx.operators.custom_ops" + +# Optional: GenAI for LLM OGA flow +python -c "import onnxruntime_genai; print('GenAI:', onnxruntime_genai.__version__)" +``` + +## Interaction Flow + +1. **Intake**: Determine what the user already has installed and what accelerator they need. Check + if `quark-env-preflight` has already run. Detect any pre-existing `onnxruntime*` variants. +2. **Plan**: Present the accelerator-specific ONNX Runtime install command, the `onnx` / + `onnxslim` / `onnxscript` commands, the C++ compiler check, and (if relevant) the GenAI add-on. + Justify each version against `install_onnxruntime.sh` and `requirements.txt`. +3. **Confirm**: Required before any package change. Show: what will be uninstalled, what will be + installed, which `--extra-index-url` will be used, and what environment will be modified. +4. **Execute**: Run uninstall (if a conflicting variant is present), then the install commands. +5. **Verify**: Run all verification commands. Report pass/fail for each, especially: + - expected EP present in `get_available_providers()`, + - custom-ops compile succeeds. + +## Recovery + +- **If `get_available_providers()` does not include the expected accelerator EP**: The CPU build of + `onnxruntime` was installed instead of the GPU build (or both variants are present). Show the + exact uninstall + reinstall commands. +- **If both `onnxruntime` and `onnxruntime-gpu` are installed**: Uninstall both (`pip uninstall -y + onnxruntime onnxruntime-gpu onnxruntime_rocm`), then reinstall only the correct variant. +- **If `import quark.onnx.operators.custom_ops` fails to compile**: Check `g++` (Linux) or VS 2022 + (Windows) is installed and on PATH; for GPU builds, check `ROCM_PATH` / `CUDA_HOME` is set. +- **If `onnx` import succeeds but Quark complains about a schema mismatch**: `onnx` version is + outside `>=1.21.0,<=1.22.0`. Reinstall to a pinned version inside the range. +- **If on ROCm 6.x and the Artifactory wheel is unreachable**: The user is off the AMD internal + network. Surface the gap — do not silently install the CPU variant. Document the ROCm-EP loss + before proceeding if the user accepts CPU fallback. +- **If Python version is wrong**: Recommend creating a new conda environment with a supported + version (3.11, 3.12, or 3.13). + +## Windows-Specific Notes + +- ROCm is not supported on Windows — only CUDA and CPU variants of ONNX Runtime are available. +- The custom-ops library compile requires Visual Studio 2022+ with the *Desktop development with + C++* workload. Use the Developer Command Prompt or set the build-tool paths via the developer + command file. +- If pip fails with long path errors when installing `onnx` / `onnxruntime`: enable Win32 long + paths via Group Policy Editor (Computer Configuration > Administrative Templates > System > + Filesystem > Enable Win32 long paths). +- WSL2 with Ubuntu is recommended as an alternative for Windows users who need ROCm. diff --git a/skills/quark-onnx-install/skill-card.md b/skills/quark-onnx-install/skill-card.md new file mode 100644 index 0000000..7ab5ea8 --- /dev/null +++ b/skills/quark-onnx-install/skill-card.md @@ -0,0 +1,13 @@ +# Skill Card + +## Description + +Install or verify the correct ONNX Runtime build (and the matching `onnx` package) for a user's accelerator backend before Quark ONNX-flow usage. Trigger for "install onnxruntime", "pip install onnxruntime", "set up onnxruntime for ROCm", "set up onnxruntime for CUDA", "onnxruntime-gpu vs onnxruntime", "onnx version m… + +## Owner + +amd (federated from [amd/Quark](https://github.com/amd/Quark)) + +## License + +MIT diff --git a/skills/quark-onnx-model-intake/.federated.json b/skills/quark-onnx-model-intake/.federated.json new file mode 100644 index 0000000..7486b83 --- /dev/null +++ b/skills/quark-onnx-model-intake/.federated.json @@ -0,0 +1,10 @@ +{ + "source": "amd-quark", + "repo": "amd/Quark", + "ref": "v0.12", + "commit": "1b229f781a1974cc742884e42d8eefc1eebb4f0a", + "path": ".claude/skills/quark-onnx-model-intake", + "license": "MIT", + "imported_at": "2026-08-05T21:07:39Z", + "implementation_path": ".claude/skills-impl/l1-atomic/onnx/quark-onnx-model-intake" +} diff --git a/skills/quark-onnx-model-intake/SKILL.md b/skills/quark-onnx-model-intake/SKILL.md new file mode 100644 index 0000000..886f3b8 --- /dev/null +++ b/skills/quark-onnx-model-intake/SKILL.md @@ -0,0 +1,356 @@ +--- +name: quark-onnx-model-intake +description: >- + Inspect a target ONNX model and prepare metadata for Quark ONNX PTQ planning. Use for `.onnx` + path validation, opset / IR version detection, input-output shape and dtype discovery, op-type + histogram, quantizable-op counting, deployment-target compatibility checks (CPU / CUDA / ROCm / + AMD NPU CNN / AMD NPU Transformer), and risk assessment. Trigger for "analyze my ONNX model", + "check this onnx model", "what opset is this", "can Quark quantize this .onnx", "is my model + NPU-compatible", "does my model already have QDQ", "is my model larger than 2 GB", or before any + ONNX quantization step that needs model facts that are missing. +--- + +# quark-onnx-model-intake + +## Catalog portability + +This standalone skill is federated from `amd/Quark` at commit `1b229f781a1974cc742884e42d8eefc1eebb4f0a`. Resolve bundled files relative to this `SKILL.md`. Repository-relative paths such as `docs/`, `examples/`, and `quark/` refer to the [pinned Quark source](https://github.com/amd/Quark/tree/1b229f781a1974cc742884e42d8eefc1eebb4f0a); use a local Quark checkout when available, otherwise consult that pinned source. + +When the workflow names one of these internal procedures, read its bundled reference before carrying out that step: + +- [`quark-workspace-validate`](references/quark-workspace-validate.md) + +## Purpose + +Validate the target `.onnx` model and extract the structural facts that an ONNX quant plan needs +to make correct quantization decisions. This step exists because different ONNX models have very +different quantization requirements — a CNN destined for AMD NPU CNN needs NHWC layout and PoF2 +scales, an LLM exported to ONNX needs MatMulNBits or SmoothQuant + GPTQ, a >2 GB model needs +external-data handling, and a model that already contains QDQ nodes (or Quark's custom-op domains) +must not be silently re-quantized. + +## Inputs + +- Model path — a single `.onnx` file, or a directory containing `model.onnx` plus its + `model.onnx_data` / external-weight blobs +- `env_context.json` for Python and accelerator constraints (CUDA major / ROCm major / NPU + presence) +- `workspace_context.json` for the validated model path +- `onnx_install_result.json` (optional) — the installed `onnxruntime*` variant determines which + execution providers are realistic deployment targets + +## Outputs: model_analysis.json + +Captures the model's graph metadata, op-type coverage, quantizable-op count, deployment-target +compatibility, and risks. Schema is shared with the Torch intake; ONNX-specific fields live under +`model.onnx_specific` and `quantization_targets.onnx_specific`. + +Schema: [`model_analysis.schema.json`](../../shared/contracts/model_analysis.schema.json) + +```json +{ + "analysis_status": "complete", + "model": { + "model_path": "/data/models/resnet50/model.onnx", + "model_type": "onnx", + "loading_class": "onnx.load", + "estimated_size_gb": 0.10, + "onnx_specific": { + "ir_version": 8, + "producer_name": "pytorch", + "producer_version": "2.5.1", + "opsets": [{"domain": "ai.onnx", "version": 17}], + "uses_external_data": false, + "external_data_files": [], + "input_layout_guess": "NCHW", + "inputs": [ + {"name": "input", "dtype": "float32", "shape": [1, 3, 224, 224]} + ], + "outputs": [ + {"name": "output", "dtype": "float32", "shape": [1, 1000]} + ], + "has_dynamic_dims": false, + "has_qdq_already": false, + "has_quark_custom_ops": false, + "architecture_guess": "cnn" + } + }, + "quantization_targets": { + "linear_layer_count": 53, + "onnx_specific": { + "op_counts": {"Conv": 53, "BatchNormalization": 53, "Relu": 49, "MaxPool": 1, "GlobalAveragePool": 1, "Gemm": 1}, + "quantizable_op_counts": {"Conv": 53, "Gemm": 1, "MatMul": 0, "Add": 16, "Mul": 0}, + "preset_candidates": ["XINT8", "A8W8", "INT8_CNN_DEFAULT"], + "exclude_defaults": [] + } + }, + "risks": [ + { + "severity": "low", + "message": "Inputs are NCHW. AMD NPU CNN target expects NHWC.", + "recovery_hint": "Run quark.onnx.tools.convert_nchw_to_nhwc before quantization." + } + ] +} +``` + +## Supported Domains and Preset Families + +Quark ONNX ships a large `DefaultConfigMapping` of named presets in +`quark/onnx/quantization/config/custom_config.py`. The intake should narrow the candidate set +based on what the model actually looks like: + +| Category | Representative presets | When to suggest | +|----------|------------------------|-----------------| +| **AMD NPU CNN (XINT8 family)** | `XINT8`, `XINT8_ADAROUND`, `XINT8_ADAQUANT`, `XINT8_WEIGHTSONLY_ADAROUND` | Model is Conv-heavy (CNN), inputs are 4D, deployment target is `npu_cnn`; requires PoF2 scales and (usually) NHWC layout | +| **AMD NPU Transformer (INT16 transformer)** | `INT16_TRANSFORMER_DEFAULT`, `INT16_TRANSFORMER_ACCURATE` | Model is MatMul/Gemm-heavy, deployment target is `npu_transformer` | +| **General CPU/GPU INT8** | `A8W8`, `A8W8_ADAROUND`, `A8W8_ADAQUANT`, `INT8_CNN_DEFAULT`, `INT8_CNN_ACCURATE` | Generic INT8 deployment on CPU/CUDA/ROCm | +| **A16W8 / wide activations** | `A16W8`, `A16W8_ADAROUND`, `A16W8_ADAQUANT` | Activations need 16-bit headroom (outlier-heavy models) | +| **Block formats (custom-ops required)** | `BFP16`, `BFP16_ADAQUANT`, `MX4`, `MX6`, `MX9`, `MXFP4`, `MXFP6`, `MXFP8`, `BF16_MIXED_BFP16`, `BF16_MIXED_MXINT8` | Modern AMD accelerators with `quark.onnx.operators.custom_ops` available | +| **Float fallbacks** | `FP16`, `BF16`, `FP16_ADAQUANT`, `BF16_ADAQUANT` | Accuracy-first; no INT quant | + +`preset_candidates` in `model_analysis.json` should list the 1–3 presets that are realistic given +architecture + deployment target; never list all of them. + +## What to Extract + +### From the ONNX model proto (without loading weights) + +- **IR version**: `model.ir_version` +- **Producer**: `model.producer_name`, `model.producer_version` +- **Opsets**: `[(opset.domain or 'ai.onnx', opset.version) for opset in model.opset_import]` — + for QDQ insertion, `ai.onnx` opset must be ≥ 13 (≥ 19 recommended for full INT4/FP8 support) +- **Custom domains**: presence of `com.amd.quark` or `com.microsoft` indicates the model has + already been processed by Quark or ORT and should not be naively re-quantized +- **External data**: `os.path.exists(model_path + '_data')` or any tensor with + `tensor.data_location == TensorProto.EXTERNAL`; total file size > 2 GB without external data is + invalid +- **Inputs / outputs**: name, dtype (from `type.tensor_type.elem_type`), shape (dim_value or + symbolic `dim_param`) +- **Dynamic dims**: any `dim_param` or `dim_value == 0` in inputs → flag as risk for NPU targets +- **Input layout guess**: 4D float input with channel dim == 3 or small power-of-2 → likely NCHW + (channel-first) or NHWC (channel-last); use the convention of the producer +- **Architecture guess**: histogram of op types — Conv-dominated → CNN; MatMul/Gemm + softmax → + Transformer; LSTM / GRU / RNN → recurrent; mix of Conv + MatMul + Attention → hybrid + +### Quantization Targets + +- **Quantizable op counts**: count `Conv`, `Gemm`, `MatMul`, `ConvTranspose`, `Add`, `Mul`, + `BatchNormalization` (will be folded into Conv during pre-processing) +- **`linear_layer_count`**: sum of `MatMul` + `Gemm` + `Conv` (the canonical "linear ops" in + ONNX) +- **`exclude_defaults`**: usually empty for ONNX (Quark doesn't have a `lm_head`-equivalent + default), but flag the final classifier `Gemm`/`MatMul` if accuracy is at risk +- **Already-quantized check**: presence of `QuantizeLinear` / `DequantizeLinear` / + `BFPQuantizeDequantize` / `MXQuantizeDequantize` / `ExtendedQuantizeLinear` → set + `model.onnx_specific.has_qdq_already = true` or `has_quark_custom_ops = true` + +### Deployment-Target Compatibility + +Cross-reference op types against the chosen target: + +- **NPU CNN** (`enable_npu_cnn=True`, XINT8): expects NHWC, PoF2 scales, per-tensor symmetric + INT8, no MatMul-based attention. Flag NCHW inputs, dynamic dims, MatMul-heavy graphs. +- **NPU Transformer** (`enable_npu_transformer=True`): QDQ on Gemm/MatMul only; flag heavy use + of unsupported ops (e.g. custom attention kernels). +- **CPU/CUDA/ROCm general**: largely permissive; flag custom-domain ops that require + matching custom-ops library at inference. +- **Block formats (BFP16 / MX / MXFP*)**: require Quark's custom-ops library compiled for the + target EP. If `onnx_install_result.json` shows the custom-ops compile failed, suppress these + presets from `preset_candidates` and emit a risk. + +### Calibration Data Considerations + +Surface what the user must prepare separately: + +- Input names + dtypes + shapes — the `CalibrationDataReader` must yield batches keyed by these + names with these dtypes +- Suggested sample count: 64–512 for CV; 128–1024 for LLM; reduce if `optimize_mem=True` is + needed +- Flag dynamic batch / seq dims that the data reader must concretize + +### Risks + +Flag anything that could cause failures downstream: + +- Model file > 2 GB **without** external data → ProtoBuf serialization will fail +- Opset < 13 → QDQ insertion may fail; suggest `onnx.version_converter.convert_version(model, 13+)` +- Dynamic dims in input shape → NPU targets and many post-processing steps don't tolerate + unconcretized dims; recommend `quark.onnx.tools.fix_shapes` +- NCHW inputs for an NPU CNN target → recommend `quark.onnx.tools.convert_nchw_to_nhwc` +- FP16-only model → some calibrators expect FP32; recommend + `quark.onnx.tools.convert_fp16_to_fp32` first +- Model already contains QDQ or Quark custom ops → do not re-quantize; suggest + `quark.onnx.tools.remove_qdq` or treat as a no-op +- Custom-domain ops present (`com.microsoft`, third-party) → flag, may require + `op_types_to_quantize` curation +- Unusual loading requirements (encrypted / crypto mode) → require the user to pass an + `onnx.ModelProto` rather than a path (see `input_check.py:137`) + +## Concrete Actions + +### Action 1: Read model metadata (NEVER load external weights) + +```bash +python3 - <<'PY' +import json, os, sys, collections +import onnx +from onnx import TensorProto + +MODEL_PATH = "" + +# load_external_data=False so we never pull the >GB tensors into memory +model = onnx.load(MODEL_PATH, load_external_data=False) + +# basic metadata +opsets = [{"domain": o.domain or "ai.onnx", "version": o.version} for o in model.opset_import] +producer = {"name": model.producer_name, "version": model.producer_version, "ir_version": model.ir_version} + +def fmt_shape(t): + return [(d.dim_value if d.dim_value else (d.dim_param or "?")) for d in t.type.tensor_type.shape.dim] + +dtype_map = {v: k for k, v in TensorProto.DataType.items()} +def fmt_dtype(t): + return dtype_map.get(t.type.tensor_type.elem_type, "UNKNOWN") + +inputs = [{"name": i.name, "dtype": fmt_dtype(i), "shape": fmt_shape(i)} for i in model.graph.input] +outputs = [{"name": o.name, "dtype": fmt_dtype(o), "shape": fmt_shape(o)} for o in model.graph.output] + +# op-type histogram +op_counts = collections.Counter(n.op_type for n in model.graph.node) + +# external data check +file_size_bytes = os.path.getsize(MODEL_PATH) +uses_external = any(init.data_location == TensorProto.EXTERNAL for init in model.graph.initializer) + +# already-quantized check +qdq_ops = {"QuantizeLinear", "DequantizeLinear"} +quark_ops = {"BFPQuantizeDequantize", "MXQuantizeDequantize", + "ExtendedQuantizeLinear", "ExtendedDequantizeLinear"} +has_qdq = any(n.op_type in qdq_ops for n in model.graph.node) +has_quark_custom = any(n.op_type in quark_ops for n in model.graph.node) + +# dynamic dim check +has_dynamic = any(isinstance(d, str) or d == "?" for inp in inputs for d in inp["shape"]) + +# architecture guess (very rough) +arch = "unknown" +if op_counts.get("Conv", 0) > 5 * op_counts.get("MatMul", 0): + arch = "cnn" +elif op_counts.get("MatMul", 0) + op_counts.get("Gemm", 0) > 5 * op_counts.get("Conv", 0): + arch = "transformer" +elif {"LSTM", "GRU", "RNN"} & set(op_counts): + arch = "recurrent" +elif op_counts.get("Conv", 0) and op_counts.get("MatMul", 0): + arch = "hybrid" + +info = { + "ir_version": producer["ir_version"], + "producer_name": producer["name"], + "producer_version": producer["version"], + "opsets": opsets, + "file_size_bytes": file_size_bytes, + "uses_external_data": uses_external, + "inputs": inputs, + "outputs": outputs, + "has_dynamic_dims": has_dynamic, + "has_qdq_already": has_qdq, + "has_quark_custom_ops": has_quark_custom, + "architecture_guess": arch, + "op_counts": dict(op_counts.most_common(30)), + "quantizable_op_counts": {k: op_counts.get(k, 0) for k in ("Conv", "Gemm", "MatMul", "ConvTranspose", "Add", "Mul", "BatchNormalization")}, +} +print(json.dumps(info, indent=2)) +PY +``` + +### Action 2: Optional — run schema and shape-inference checks + +```bash +python3 - <<'PY' +import onnx +MODEL_PATH = "" +model = onnx.load(MODEL_PATH, load_external_data=False) +onnx.checker.check_model(model, full_check=False) # raises on schema errors +inferred = onnx.shape_inference.infer_shapes(model) # may reveal shape issues +print("checker: OK, shapes inferred") +PY +``` + +### Action 3: Check model size on disk (catches the 2 GB ProtoBuf limit early) + +```bash +du -sh /path/to/model.onnx +ls -lh /path/to/model.onnx* # picks up model.onnx_data +``` + +### Action 4: Present summary table to user + +```text +ONNX Model Analysis: + Model path: + File size: external-data: Yes/No + IR / Opset: ir=, ai.onnx= (custom domains: ) + Architecture: + Inputs: : + Outputs: : + Quantizable ops: Conv=, MatMul=, Gemm= (linear_layer_count=) + Already QDQ: Yes/No Already Quark custom ops: Yes/No + Preset candidates: [] + Risks: +``` + +## Rules + +- **Run or reference `quark-workspace-validate` first** to confirm `.onnx` and adjacent + `.onnx_data` paths are valid before attempting to parse the model. +- **Do not load external data.** Pass `load_external_data=False` to `onnx.load`. The intake reads + graph metadata only; loading multi-GB tensor blobs is the quantizer's job. +- **Refuse to silently re-quantize.** If `has_qdq_already` or `has_quark_custom_ops` is true, + emit a high-severity risk and require the user to confirm intent (likely they want + `quark.onnx.tools.remove_qdq` first, or they meant to quantize a different file). +- **Preserve ambiguity** when a model reference could be a single `.onnx` or a directory of + external-data shards. Note both possibilities and let the user resolve. +- **Filter `preset_candidates` by deployment target and custom-op availability.** Never suggest + BFP/MX presets when `onnx_install_result.json` shows the custom-ops library failed to compile. + Never suggest XINT8 outside an NPU CNN target. +- **Surface new environment constraints** (e.g. model requires opset upgrade, or NHWC conversion + before NPU CNN) by recording them in `model_analysis.json` under `risks` and asking + `the matching public quark-onnx skill` to add them to `session_context.json`'s `open_questions`. Do not write + directly to `env_context.json`. +- **Never call `quantize_static` / `ModelQuantizer.quantize_model` here.** Intake is read-only. + +## Interaction Flow + +1. **Confirm model source**: Is the path a single `.onnx` or a directory? Are there + `.onnx_data` shards present? Check if `quark-workspace-validate` already confirmed the path. +2. **Read metadata**: Run Action 1 (and optionally Action 2). Capture opsets, inputs, outputs, + op counts, external-data state, already-QDQ flags. +3. **Classify architecture**: CNN / transformer / recurrent / hybrid based on op histogram. +4. **Cross-reference deployment target**: From `session_context.json.constraints.deployment_target` + (or ask if missing), narrow `preset_candidates` to viable options. +5. **Identify risks**: NCHW vs NHWC, dynamic dims, > 2 GB without external data, opset too low, + already-quantized, FP16-only, custom-domain ops, missing custom-ops compile for BFP/MX. +6. **Present summary table** to the user. +7. **Emit**: Write `model_analysis.json`. Surface any new constraints back to `the matching public quark-onnx skill` + so they land in `session_context.json`. + +## Recovery + +- If `analysis_status: "partial"` — some facts were extracted but the model could not be fully + inspected. Common causes: external-data files missing, ProtoBuf > 2 GB without external data, + or `onnx.checker` raised a schema error. +- If `has_qdq_already == true` or `has_quark_custom_ops == true` — do **not** proceed to PTQ + planning. Recommend `python -m quark.onnx.tools.remove_qdq` (or equivalent) and re-run intake + on the cleaned model. +- If `opsets[ai.onnx] < 13` — recommend + `onnx.version_converter.convert_version(model, 13)` (or 19+ for INT4/FP8) and re-run intake. +- If `has_dynamic_dims == true` for an NPU target — recommend + `python -m quark.onnx.tools.fix_shapes --input model.onnx --output model_fixed.onnx + --input_shape "input:1,3,224,224"` and re-run intake. +- If `input_layout_guess == "NCHW"` and target is `npu_cnn` — recommend + `python -m quark.onnx.tools.convert_nchw_to_nhwc` and re-run intake. +- If the model is FP16-only and the chosen calibrator needs FP32 — recommend + `python -m quark.onnx.tools.convert_fp16_to_fp32` and re-run intake. +- If the architecture is "unknown" — present the op histogram to the user and ask which target + they intend; do not guess a preset. diff --git a/skills/quark-onnx-model-intake/references/quark-workspace-validate.md b/skills/quark-onnx-model-intake/references/quark-workspace-validate.md new file mode 100644 index 0000000..0000ebc --- /dev/null +++ b/skills/quark-onnx-model-intake/references/quark-workspace-validate.md @@ -0,0 +1,113 @@ +--- +name: quark-workspace-validate +description: > + Validate workspace shape, model paths, output directories, and repo structure before downstream Quark skills proceed. + Use this skill whenever a skill needs confirmed file paths, when the user provides a model path or output directory, + when you need to distinguish a local model from a HuggingFace ID, or when any path assumption is unverified. + Also trigger for "where should I put outputs", "is my model path right", or before model-intake/export steps. +layer: l0-foundation +primary_artifact: workspace_context.json +source_knowledge: + - examples/torch/language_modeling/llm_ptq/README.md + - examples/torch/language_modeling/llm_ptq/quantize_quark.py +--- + +# quark-workspace-validate + +## Purpose + +Validate that file paths, model references, and output locations actually exist and are accessible before a downstream skill depends on them. This prevents silent failures where a PTQ run starts, spends time loading a model, and then crashes because the output directory does not exist or the model path has a typo. + +## Inputs + +- Model path and output directory from the user (typically forwarded by `the matching public quark-torch skill`) + +## Outputs: workspace_context.json + +Records validated model paths, output directory, and repo locations. + +Schema: [`workspace_context.schema.json`](https://github.com/amd/Quark/blob/1b229f781a1974cc742884e42d8eefc1eebb4f0a/.claude/skills-impl/shared/contracts/workspace_context.schema.json) + +```json +{ + "model_path": "/models/Qwen3-8B", + "model_source": "local", + "trust_remote_code_required": false, + "output_dir": "./output/qwen3-8b-fp8", + "output_dir_exists": false, + "output_dir_parent_writable": true, + "disk_space_gb_free": 250, + "quark_script_path": null +} +``` + +`workspace_context.json` is for path facts only. Unresolved questions (e.g., ambiguous local-vs-HuggingFace reference) belong in `session_context.json` (owned by `the matching public quark-torch skill`). + +## What to Validate + +### Model Path + +A model reference can be one of three things: + +1. **Local directory** — contains `config.json`, `*.safetensors` or `*.bin` files (e.g., `/models/Llama-2-7b-hf/`) +2. **HuggingFace ID** — format `org/model-name` (e.g., `Qwen/Qwen3-8B`) +3. **Ambiguous** — could be either (e.g., `./qwen3-8b` might be a local dir or a typo) + +For local paths, check: + +- Directory exists +- `config.json` is present (minimum requirement for HuggingFace-compatible model) +- At least one weight file exists (`.safetensors`, `.bin`, or `.pth`) +- Read permissions are adequate + +For HuggingFace IDs, note: + +- Cannot fully validate without network access +- Check format: should contain `/` separator +- Record whether `trust_remote_code` will be needed (some models like DeepSeek VL v2 require it) + +### Output Directory + +- Check that the parent directory exists and is writable +- If the output directory itself does not exist, note that it will be created (not an error) +- Warn if the directory already contains files (risk of overwriting previous results) + +### Quark Repository (if applicable) + +- If the user is running from source, check for `quantize_quark.py` at the expected location: `examples/torch/language_modeling/llm_ptq/quantize_quark.py` +- Check that `requirements.txt` dependencies are likely installed + +## Validation Commands + +```bash +# Check model directory +ls -la /path/to/model/config.json 2>/dev/null +ls /path/to/model/*.safetensors 2>/dev/null + +# Check output directory parent +test -d /path/to/output/.. && echo "parent exists" +test -w /path/to/output/.. && echo "parent writable" + +# Check disk space (rough) +df -h /path/to/output/.. +``` + +## Rules + +- **Only validate existence, accessibility, and shape.** Do not analyze model internals (that is `quark-torch-model-intake`) or choose install paths (that is `quark-install`). +- **Never invent replacement paths.** If `/data/models/llama` does not exist, say so — do not suggest `/data/models/llama-2` as an alternative unless the user asks. +- **Preserve ambiguity explicitly.** When a reference like `meta-llama/Llama-2-7b` could be local or remote, say: "This looks like a HuggingFace model ID. If you meant a local directory, the path does not exist at `./meta-llama/Llama-2-7b`." +- **Check disk space** when the downstream task is PTQ or export — quantized models can be large. + +## Interaction Flow + +1. **Collect**: Gather the paths the downstream skill needs — model path, output directory, any referenced config files. +2. **Validate**: Run existence and permission checks. Present results as a checklist with pass/fail for each item. +3. **Clarify**: For ambiguous references, ask the user. For missing paths, report the exact issue. +4. **Emit**: Write confirmed facts to `workspace_context.json`. Hand any unresolved items back to `the matching public quark-torch skill` so they land in `session_context.json`'s `open_questions`. + +## Recovery + +- If a required path is missing, report the exact path tested and what was expected. Suggest the smallest fix: "Directory `/data/models/llama` does not exist. Did you mean `/data/models/Llama-2-7b-hf`?" +- If a model reference is ambiguous between local and HuggingFace, keep it unresolved and ask — do not guess. +- If disk space is low for the intended output, warn with the estimated size needed. diff --git a/skills/quark-onnx-model-intake/skill-card.md b/skills/quark-onnx-model-intake/skill-card.md new file mode 100644 index 0000000..f7175db --- /dev/null +++ b/skills/quark-onnx-model-intake/skill-card.md @@ -0,0 +1,13 @@ +# Skill Card + +## Description + +Inspect a target ONNX model and prepare metadata for Quark ONNX PTQ planning. Use for `.onnx` path validation, opset / IR version detection, input-output shape and dtype discovery, op-type histogram, quantizable-op counting, deployment-target compatibility checks (CPU / CUDA / ROCm / AMD NPU CNN / AMD NPU Transformer)… + +## Owner + +amd (federated from [amd/Quark](https://github.com/amd/Quark)) + +## License + +MIT diff --git a/skills/quark-onnx-ptq/.federated.json b/skills/quark-onnx-ptq/.federated.json new file mode 100644 index 0000000..bb26913 --- /dev/null +++ b/skills/quark-onnx-ptq/.federated.json @@ -0,0 +1,10 @@ +{ + "source": "amd-quark", + "repo": "amd/Quark", + "ref": "v0.12", + "commit": "1b229f781a1974cc742884e42d8eefc1eebb4f0a", + "path": ".claude/skills/quark-onnx-ptq", + "license": "MIT", + "imported_at": "2026-08-05T21:07:39Z", + "implementation_path": ".claude/skills-impl/l2-workflows/onnx/quark-onnx-ptq-workflow" +} diff --git a/skills/quark-onnx-ptq/SKILL.md b/skills/quark-onnx-ptq/SKILL.md new file mode 100644 index 0000000..8cc4732 --- /dev/null +++ b/skills/quark-onnx-ptq/SKILL.md @@ -0,0 +1,344 @@ +--- +name: quark-onnx-ptq +description: >- + End-to-end ONNX PTQ workflow for AMD Quark — for `.onnx` input models (with optional sibling + `.onnx_data` external-weights file). Use when the user wants a complete ONNX-to-ONNX pipeline: + model intake, quantization planning, calibration-script generation, manifest, and confirmed + execution. Trigger for "quantize my .onnx", "run ONNX PTQ end to end", "full ONNX quantization + pipeline", "quantize yolov8/resnet50/yolo_nas with XINT8/A8W8/BFP16/MXFP*", "weights-only INT4 + for my .onnx LLM", or any request that spans more than one ONNX PTQ step. Not for HuggingFace / + safetensors / PyTorch input models — use quark-torch-ptq instead. +--- + +# quark-onnx-ptq + +## Catalog portability + +This standalone skill is federated from `amd/Quark` at commit `1b229f781a1974cc742884e42d8eefc1eebb4f0a`. Resolve bundled files relative to this `SKILL.md`. Repository-relative paths such as `docs/`, `examples/`, and `quark/` refer to the [pinned Quark source](https://github.com/amd/Quark/tree/1b229f781a1974cc742884e42d8eefc1eebb4f0a); use a local Quark checkout when available, otherwise consult that pinned source. + +When the workflow names one of these internal procedures, read its bundled reference before carrying out that step: + +- [`quark-onnx-quant-plan`](references/quark-onnx-quant-plan.md) + +> 📘 **Quick-start example — read this first.** +> A fully worked end-to-end walkthrough is available at **[`example-xint8-yolov8n.md`](example-xint8-yolov8n.md)** (XINT8 quantization of YOLOv8n for AMD NPU CNN deployment, calibrated on COCO val2017). It is the fastest way to see exactly what this workflow produces — open it alongside this SKILL.md before running anything. + +## Purpose + +Chain the ONNX PTQ path — model intake → quantization planning → script + manifest generation → confirmed execution — while keeping the user informed at each checkpoint. This workflow orchestrates the atomic ONNX skills so the user does not have to manually chain them. Unlike the Torch flow there is no single shipped `quantize_quark.py` for ONNX; the workflow generates a small standalone Python script in the user's working directory that imports from `quark.onnx`, then runs that script after the user confirms. + +📂 **Worked example:** see [`example-xint8-yolov8n.md`](example-xint8-yolov8n.md) for the full YOLOv8n + XINT8 + NPU-CNN walkthrough referenced throughout the steps below. + +## Inputs + +- Input `.onnx` model path (with optional sibling `.onnx_data` external-weights file) +- Calibration data: folder of representative samples (or `CalibrationDataReader` Python class) +- Output `.onnx` path (and optional `.onnx_data` if `use_external_data_format=True`) +- User goal: scheme (XINT8 / A8W8 / A16W8 / BF16 / BFP16 / `MX*` / `MXFP*`), deployment target (CPU / CUDA / ROCm / AMD NPU CNN / AMD NPU Transformer), accuracy target +- `session_context.json` with `constraints.backend = "onnx"` for user goal and constraints +- `env_context.json` for hardware / execution-provider facts +- `workspace_context.json` for validated paths +- `onnx_install_result.json` and `quark_install_result.json` to confirm runtime is ready + +## Outputs: run_manifest.yaml + +Records the generated calibration/quantization script path, the exact `python3` invocation, and the resolved `QConfig`. Side artifacts: the generated script in the user's working directory, the quantized `.onnx` (and `.onnx_data` if external) in the user's output directory. The manifest is built in Step 3. + +Schema: [`run_manifest.schema.json`](https://github.com/amd/Quark/blob/1b229f781a1974cc742884e42d8eefc1eebb4f0a/.claude/skills-impl/shared/contracts/run_manifest.schema.json) + +## Interaction Flow + +1. **Intake** — call `quark-onnx-model-intake` to produce `model_analysis.json` +2. **Plan** — call `quark-onnx-quant-plan` to produce `quant_plan.json` +3. **Manifest** — generate a standalone quantization script in the user's working directory + `run_manifest.yaml`, then stop for user approval +4. **Execute** — run the confirmed script and report the artifact paths + +## CRITICAL RULES + +1. **NEVER call `ModelQuantizer.quantize_model(...)` directly from this workflow.** Always generate a standalone script the user reviews first. +2. **NEVER skip a step.** Even if the user provides all details upfront, execute each step in order. +3. **STOP at every checkpoint** and wait for user confirmation before continuing. +4. **Show concrete output** at each step — tables, JSON, the full script body, the exact command — not just prose descriptions. +5. **NEVER modify Quark's own source code, examples, or tutorials.** The Quark repo (`quark/`, `examples/`, `tutorials/`, `tools/`, `docs/`, `tests/`) is read-only from this workflow's perspective. See *Upstream Quark Code is Read-Only* below. +6. **NEVER silently fall back to CPU execution.** If the user requested CUDA/ROCm and the provider is unavailable, stop and surface the gap to `quark-onnx-install` / `quark-onnx-debug` — do not quietly degrade. + +## Upstream Quark Code is Read-Only + +The Quark repository is the upstream source of truth. This workflow may read it freely (config sources, example scripts, tutorial notebooks, custom-op headers) but must not write into it. That includes: + +- No edits to `quark/` package source. +- No edits to anything under `examples/onnx/` — including `examples/onnx/yolo_quantization/quantize_yolo.py`. Even small "just to make the script accept my preprocessing" patches are forbidden, because they make the run irreproducible against a clean Quark install. +- No edits to anything under `tutorials/onnx/` — the Ryzen AI tutorials for YOLOv8 / ResNet-50 / image classification are reference material only. +- No edits to `tools/`, `docs/`, `tests/`, or `pyproject.toml` / `requirements.txt`. + +**If a shipped example or tutorial as-is does not cover what the user needs** (different model, different preprocessing, different exclude list, different calibration source), write a fresh standalone script in the user's working directory (or `/tmp/`) that imports from `quark.onnx`. Pattern: + +```python +# user_workspace/my_onnx_ptq.py — NOT inside the Quark repo +from quark.onnx import ModelQuantizer, QConfig, QLayerConfig, XInt8Spec, CLEConfig +from onnxruntime.quantization.calibrate import CalibrationDataReader +# ... user-specific data reader + QConfig the shipped example doesn't cover ... +``` + +Then reference *that* script in the `run_manifest.yaml` instead of a modified example. The shipped scripts stay untouched, the user's customization is local to their workspace, and the run remains reproducible against any Quark version. + +If the user actually needs an upstream Quark change (a new preset, a new algorithm config), surface it as a Quark contribution — do not silently patch their local checkout. + +## Required Artifact Flow + +```text +Step 1 (Intake) ──► model_analysis.json +Step 2 (Plan) ──► quant_plan.json +Step 3 (Manifest) ──► run_manifest.yaml + user_workspace/_ptq.py +Step 4 (Execute) ──► quantized .onnx (+ .onnx_data if external) (only after user says yes) +``` + +--- + +## Step 1: Model Intake + +**Goal**: Analyze the `.onnx` graph and produce `model_analysis.json`. Hand off to `quark-onnx-model-intake`. + +### Actions + +1. **Confirm the input path.** It must end in `.onnx`. If a sibling `.onnx_data` exists, both must move together. +2. **Run `quark-onnx-model-intake`.** That skill produces: + - opset / IR version + - input / output names, shapes, dtypes + - op-type histogram and quantizable-op count + - whether QDQ nodes are already present (if yes — model is already quantized; warn and stop) + - external-data status (model.onnx_data present? total bytes? >2 GB?) + - deployment-target compatibility (CPU / CUDA / ROCm / NPU CNN / NPU Transformer) +3. **Surface risks:** + - Already quantized → stop, do not re-quantize. + - opset < 13 → recommend upgrading to opset 17+ for best support. + - Dynamic input shapes → calibration data reader must produce matching shapes; warn if shapes vary. + - >2 GB model → external data must be handled; set `use_external_data_format=True` later. + - NPU CNN target → only XINT8 / A8W8 are gated as supported; warn early. + +### Output to Show User + +Present a summary table: + +```text +Model Analysis: + Model path: ./models/yolov8n.onnx + Opset / IR: 17 / 8 + Input: images, [1, 3, 640, 640], float32 + Output: output0, [1, 84, 8400], float32 + Total ops: ~226 nodes (Conv, MatMul, Add, Mul, Sigmoid, Concat, …) + Quantizable ops: ~118 (Conv + MatMul + …) + QDQ already? No + External data: No (6.2 MB inline) + Risks: None + NPU CNN target: Compatible (Conv-heavy, no unsupported ops) +``` + +### >>> CHECKPOINT 1: Confirm model analysis is correct before continuing + +--- + +## Step 2: Quantization Plan + +**Goal**: Build `quant_plan.json` from the model analysis and user's stated preferences. Hand off to `quark-onnx-quant-plan`. + +### Actions + +1. **Determine the preset.** If the user stated one (e.g., "XINT8"), use it. Otherwise, recommend based on deployment target + priority: + + | Deployment | Priority | Recommended Preset | Algorithm | + |------------|----------|--------------------|-----------| + | AMD NPU CNN (Ryzen AI) | Best accuracy | `XINT8` + `EnableNPUCnn=True` | `CLE` (optional `AdaRound`) | + | AMD NPU Transformer | Best accuracy | `BFP16` | none | + | CPU general | Smallest model | `A8W8` | `CLE` (optional) | + | CPU general | Best accuracy | `A16W8` | `MinMax` | + | CUDA / ROCm | Best accuracy | `BF16` | none | + | CUDA / ROCm | High throughput | `BFP16` / `MXFP4` | none | + | Any | Recover lost accuracy | + `AdaRound` / `AdaQuant` | as additional algorithm | + +2. **Fill the decision table.** Show ALL decisions with defaults: + + | Decision | Value | Reason | + |----------|-------|--------| + | `preset` | `XINT8` | User requested XINT8 (NPU CNN) | + | `activation_spec` | `XInt8Spec()` | Matches preset | + | `weight_spec` | `XInt8Spec()` | Matches preset | + | `calibration_method` | `MinMax` (default) | Standard for XINT8 | + | `algo_config` | `[CLEConfig()]` | Improves XINT8 accuracy on Conv networks | + | `EnableNPUCnn` | `True` | XINT8 + NPU CNN deployment | + | `use_external_data_format` | `False` | Model < 2 GB | + | `exclude_nodes` / `exclude_subgraphs` | `[]` | None identified | + | `calibration_data_path` | `./calib_data/` | User-provided folder of representative samples | + | `num_calib_data` | `100` | Standard default for vision | + | `batch_size` | `1` | Safe default | + | `evaluation_intent` | `smoke` | Quick mAP / Prec@1 check after quantization | + +3. **Map deployment-target gates explicitly.** Note any preset+target combinations that are unsupported (e.g., BFP16 on NPU CNN). Surface them, do not silently change the user's choice. + +4. **Ask the user if they want to change anything.** + +### >>> CHECKPOINT 2: User MUST confirm or adjust the plan before continuing + +Wait for the user to say "ok", "confirm", "looks good", "continue", or similar. If they request changes (e.g., "add AdaRound", "raise calib data to 500"), update the table and re-present. + +--- + +## Step 3: Manifest Generation + +**Goal**: Translate the confirmed plan into a runnable standalone script in the user's working directory and a `run_manifest.yaml`. Do not write into the Quark repo. + +### Actions + +1. **Decide the script path.** Place it in the user's working directory, e.g. `./_ptq.py`. Never write under `examples/onnx/` or `tutorials/onnx/`. + +2. **Build the script body.** Three pieces: + + a. **`CalibrationDataReader`** — pick the right reader for the input modality. Vision models follow the pattern from `tutorials/onnx/ryzen_ai/yolov8/` and `tutorials/onnx/image_classification/`: + + ```python + from onnxruntime.quantization.calibrate import CalibrationDataReader + import onnxruntime as ort + import numpy as np, os, cv2 + + class ImageDataReader(CalibrationDataReader): + def __init__(self, calib_folder, model_path, hw=(640, 640)): + sess = ort.InferenceSession(model_path, providers=["CPUExecutionProvider"]) + self.input_name = sess.get_inputs()[0].name + self.data = self._load(calib_folder, *hw) + self.iter = None + def _load(self, folder, h, w): + out = [] + for f in sorted(os.listdir(folder)): + if not f.lower().endswith((".jpg", ".jpeg", ".png")): continue + img = cv2.imread(os.path.join(folder, f)) + img = cv2.resize(img, (w, h)) + arr = img.transpose(2, 0, 1).astype(np.float32) / 255.0 + out.append(np.expand_dims(arr, 0)) + return out + def get_next(self): + if self.iter is None: + self.iter = iter([{self.input_name: d} for d in self.data]) + return next(self.iter, None) + def rewind(self): self.iter = None + ``` + + b. **`QConfig`** built from the plan: + + ```python + from quark.onnx import ( + ModelQuantizer, QConfig, QLayerConfig, + XInt8Spec, Int8Spec, Int16Spec, BFloat16Spec, BFP16Spec, + CLEConfig, AdaRoundConfig, AdaQuantConfig, CalibMethod, + ) + activation_spec = XInt8Spec() + weight_spec = XInt8Spec() + algo_config = [CLEConfig()] + config = QConfig( + global_config=QLayerConfig(activation=activation_spec, weight=weight_spec), + algo_config=algo_config, + EnableNPUCnn=True, + use_external_data_format=False, + exclude=[], + ) + ``` + + c. **Driver** that wires the data reader + config + I/O paths: + + ```python + dr = ImageDataReader("./calib_data", "./models/yolov8n.onnx") + ModelQuantizer(config).quantize_model( + "./models/yolov8n.onnx", + "./models/yolov8n_xint8.onnx", + dr, + ) + ``` + +3. **Write the script to disk** at the chosen path. Print the *full* script body back to the user so they can see exactly what will run — never just say "generated a script". + +4. **Build the exact command:** + + ```bash + python3 ./yolov8n_ptq.py + ``` + +5. **Show expected output layout:** + + ```text + ./models/ + ├── yolov8n.onnx (input, unchanged) + └── yolov8n_xint8.onnx (quantized output) + ./yolov8n_ptq.py (generated by this workflow) + ./run_manifest.yaml (this workflow's manifest) + ``` + + If `use_external_data_format=True`, also expect `yolov8n_xint8.onnx_data` next to the `.onnx`. + +### >>> CHECKPOINT 3: Show the script body and the command, then ask "shall I run this?" + +Do NOT proceed to execution unless the user explicitly confirms. Acceptable confirmations: "yes", "run it", "go", "execute", or similar. + +If the user says "no" or wants changes, go back to the relevant step (most commonly back to Step 2 to change preset / algorithm / exclude lists). + +--- + +## Step 4: Execute PTQ + +**Goal**: Run the generated script and report results. + +### Precondition + +**This step runs ONLY after the user explicitly confirms in Step 3.** + +### Actions + +1. **Create the output directory if needed:** + + ```bash + mkdir -p ./models + ``` + +2. **Run the generated script.** Monitor for the common ONNX-side failures: + - ONNX Runtime EP not available → hand off to `quark-onnx-install` / `quark-onnx-debug`. + - Custom-op library load failure (`BFPQuantizeDequantize`, `MXQuantizeDequantize`, `Extended*`) → hand off to `quark-onnx-debug`. + - OOM during calibration → suggest reducing `num_calib_data`, reducing batch size, or moving calibration to CPU (`OptimDevice="cpu"`). + - External-data not found → confirm `.onnx_data` sits next to `.onnx`. + - AdaRound / AdaQuant diverged → suggest `EarlyStop=True`, lower learning rate, more iterations. + - Calibration shapes mismatch → confirm the data reader produces the model's declared input shape. + +3. **After completion, verify outputs exist:** + + ```bash + ls -lh ./models/yolov8n_xint8.onnx* + ``` + +4. **Report results:** + + ```text + Quantization complete: + Output: ./models/yolov8n_xint8.onnx + Model size: ~3.5 MB (input was 12.3 MB → ~3.5× smaller) + Format: ONNX (QDQ inserted, com.amd.quark custom ops where applicable) + External data: No + ``` + +### Error Recovery + +- If quantization fails, do NOT retry blindly. Report the error verbatim and hand off to `quark-onnx-debug`. +- For OOM, suggest: reduce `num_calib_data` first, then drop `batch_size` to 1, then move calibration to CPU. +- For EP issues, do not "fix" by silently switching providers — surface the gap and let `quark-onnx-install` resolve it. + +--- + +## Complete Example + +⭐ **Recommended starting point for new users.** + +The full end-to-end walkthrough — XINT8 quantization of **YOLOv8n** for AMD NPU CNN deployment, calibrated on COCO val2017 — is in **[`example-xint8-yolov8n.md`](example-xint8-yolov8n.md)** sitting next to this SKILL.md. It shows every checkpoint (intake → plan → manifest → execute) with real numbers, the generated script, and the resulting quantized model layout. Mirror it for your own CNN. + +## Recovery + +- If any upstream artifact is missing, stop and name the missing producer skill. Do not improvise a partial artifact. +- If the workflow reaches a blocked state (e.g., model already quantized, preset incompatible with deployment target, calibration data not in the model's input shape), report the blocker and suggest the specific fix. +- If execution fails, report the error with diagnostic context and hand off to `quark-onnx-debug` rather than attempting ad-hoc patches. +- If the user wants to change a decision mid-workflow (e.g., switch from XINT8 to BFP16 after seeing model analysis), go back to the relevant step — do not restart from scratch. +- If the user pasted a Torch traceback or a HuggingFace path, do not try to repair-route — hand off to `the matching public quark-torch skill` and stop. diff --git a/skills/quark-onnx-ptq/example-xint8-yolov8n.md b/skills/quark-onnx-ptq/example-xint8-yolov8n.md new file mode 100644 index 0000000..e098414 --- /dev/null +++ b/skills/quark-onnx-ptq/example-xint8-yolov8n.md @@ -0,0 +1,154 @@ +# Complete Example: XINT8 Quantization of YOLOv8n for AMD NPU CNN + +End-to-end walkthrough of `quark-onnx-ptq` for an XINT8 PTQ run on the YOLOv8n object-detection model, targeting AMD NPU CNN (Ryzen AI) deployment. Calibration source: COCO `val2017`. + +The workflow generates a standalone script in the user's working directory (`./yolov8n_ptq.py`) and never modifies the shipped `examples/onnx/yolo_quantization/quantize_yolo.py` or the `tutorials/onnx/ryzen_ai/yolov8/` notebook — both stay read-only references. + +```text +User: "Quantize ./models/yolov8n.onnx with XINT8 for Ryzen AI deployment, + calibrate on ./calib_data (COCO val2017 subset), output to ./models/yolov8n_xint8.onnx" + +─── Step 1: Model Intake ─── +[Run quark-onnx-model-intake against ./models/yolov8n.onnx] + +Model Analysis: + Model path: ./models/yolov8n.onnx + Opset / IR: 17 / 8 + Input: images, [1, 3, 640, 640], float32 + Output: output0, [1, 84, 8400], float32 + Total ops: ~226 nodes (Conv, MatMul, Add, Mul, Sigmoid, SiLU, Concat, …) + Quantizable ops: ~118 (Conv + MatMul + …) + QDQ already? No + External data: No (12.3 MB inline) + Risks: None + NPU CNN target: Compatible (Conv-heavy, no unsupported ops) + +>>> Does this look correct? (confirm to continue) + +─── Step 2: Quantization Plan ─── +[Run quark-onnx-quant-plan with the analysis + user's stated XINT8 + NPU CNN target] + +Decision Table: + | Decision | Value | Reason | + |----------------------------|-------------------|-----------------------------------| + | preset | XINT8 | User requested XINT8 (NPU CNN) | + | activation_spec | XInt8Spec() | Matches preset | + | weight_spec | XInt8Spec() | Matches preset | + | calibration_method | MinMax (default) | Standard for XINT8 | + | algo_config | [CLEConfig()] | Improves XINT8 accuracy on Conv | + | EnableNPUCnn | True | XINT8 + NPU CNN deployment | + | use_external_data_format | False | Model < 2 GB | + | exclude_nodes | [] | None identified | + | calibration_data_path | ./calib_data | User-provided COCO val2017 subset | + | num_calib_data | 100 | Standard default for vision | + | batch_size | 1 | Safe default | + | evaluation_intent | smoke | Quick mAP check after quant | + +>>> Confirm this plan? Any changes? (e.g. add AdaRound, raise num_calib_data, add exclude_nodes) + +─── Step 3: Manifest ─── +[Generate ./yolov8n_ptq.py and ./run_manifest.yaml in the user's working directory] + +Generated script (./yolov8n_ptq.py) — full body shown: + + import os + import cv2 + import numpy as np + import onnxruntime as ort + from onnxruntime.quantization.calibrate import CalibrationDataReader + + from quark.onnx import ( + ModelQuantizer, QConfig, QLayerConfig, + XInt8Spec, CLEConfig, + ) + + INPUT_MODEL = "./models/yolov8n.onnx" + OUTPUT_MODEL = "./models/yolov8n_xint8.onnx" + CALIB_FOLDER = "./calib_data" + NUM_CALIB = 100 + HW = (640, 640) + + class YoloImageDataReader(CalibrationDataReader): + def __init__(self, calib_folder, model_path): + sess = ort.InferenceSession(model_path, providers=["CPUExecutionProvider"]) + self.input_name = sess.get_inputs()[0].name + self.data = self._load(calib_folder) + self.iter = None + def _load(self, folder): + h, w = HW + out = [] + files = sorted(f for f in os.listdir(folder) + if f.lower().endswith((".jpg", ".jpeg", ".png"))) + for f in files[:NUM_CALIB]: + img = cv2.imread(os.path.join(folder, f)) + img = cv2.resize(img, (w, h)) + arr = img.transpose(2, 0, 1).astype(np.float32) / 255.0 + out.append(np.expand_dims(arr, 0)) + return out + def get_next(self): + if self.iter is None: + self.iter = iter([{self.input_name: d} for d in self.data]) + return next(self.iter, None) + def rewind(self): + self.iter = None + + def main(): + activation_spec = XInt8Spec() + weight_spec = XInt8Spec() + config = QConfig( + global_config=QLayerConfig(activation=activation_spec, weight=weight_spec), + algo_config=[CLEConfig()], + EnableNPUCnn=True, + use_external_data_format=False, + exclude=[], + ) + dr = YoloImageDataReader(CALIB_FOLDER, INPUT_MODEL) + ModelQuantizer(config).quantize_model(INPUT_MODEL, OUTPUT_MODEL, dr) + print(f"Wrote {OUTPUT_MODEL}") + + if __name__ == "__main__": + main() + +Command: + python3 ./yolov8n_ptq.py + +Expected output layout: + ./models/ + ├── yolov8n.onnx (input, unchanged) + └── yolov8n_xint8.onnx (quantized output, ~3.5 MB) + ./yolov8n_ptq.py (generated by this workflow) + ./run_manifest.yaml (this workflow's manifest) + +>>> Shall I run this? + +─── Step 4: Execute ─── +[Running quantization...] +[CLE pass complete] +[Calibration: 100/100 samples] +[QDQ insertion complete] +[Wrote ./models/yolov8n_xint8.onnx] + +Quantization complete: + Output: ./models/yolov8n_xint8.onnx + Model size: ~3.5 MB (input was 12.3 MB → ~3.5× smaller) + Format: ONNX (QDQ inserted, com.amd.quark custom ops where applicable) + External data: No + +For accuracy numbers (mAP@0.5, mAP@0.5:0.95), run the YOLOv8 evaluator separately — +this workflow does not run end-task accuracy benchmarks. Expected ballpark per +tutorials/onnx/ryzen_ai/yolov8/onnx_ryzen_ai_yolov8_tutorial.ipynb: + + | Metric | Float | Quantized | + |---------------|----------|-----------| + | Model Size | 12.26 MB | ~3.5 MB | + | mAP@0.5:0.95 | 0.265 | ~0.206 | + | mAP@0.5 | 0.341 | ~0.269 | +``` + +## Variants you can request mid-workflow + +- **Different preset** (e.g. `A8W8`, `A16W8`, `BF16`, `BFP16`) — restate at Step 2; the planner will swap `*Spec()` and any `EnableNPUCnn` / extras (`AlignSlice`, `FoldRelu`, `AlignConcat`, `AlignEltwiseQuantType`). +- **Recover lost accuracy** — add `AdaRoundConfig(...)` or `AdaQuantConfig(...)` to `algo_config` at Step 2; the planner will pick reasonable `data_size` / `num_iterations` / `learning_rate` defaults (mirrors `DEFAULT_ADAROUND_PARAMS` in `tutorials/onnx/ryzen_ai/yolov8/`). +- **Exclude specific nodes** (e.g. detection head Concats) — pass `exclude_nodes="/model.22/Concat_5"` at Step 2; the planner adds them to the script's `exclude=[...]` list. Subgraph exclusions follow the `([start_nodes], [end_nodes])` form from `examples/onnx/yolo_quantization/quantize_yolo.py`. +- **YOLO-NAS / YOLOX instead of YOLOv8** — same workflow, different input model and matching data reader. The shipped `examples/onnx/yolo_quantization/quantize_yolo.py` covers the YOLO-NAS / YOLOX export and dataset-loader patterns; the generated script imports `quark.onnx` directly rather than patching that file. +- **Bigger model (>2 GB)** — at Step 2 set `use_external_data_format=True`; the generated script will write `*.onnx_data` alongside the `.onnx`. diff --git a/skills/quark-onnx-ptq/references/quark-onnx-quant-plan.md b/skills/quark-onnx-ptq/references/quark-onnx-quant-plan.md new file mode 100644 index 0000000..0883ad4 --- /dev/null +++ b/skills/quark-onnx-ptq/references/quark-onnx-quant-plan.md @@ -0,0 +1,348 @@ +--- +name: quark-onnx-quant-plan +description: > + Build a Quark ONNX PTQ quantization plan from `model_analysis.json` and user intent. Use when the + user needs preset selection (XINT8 / A8W8 / A16W8 / BF16 / BFP16 / MX* / MXFP* …), + calibration method choice (MinMax / Entropy / Percentile / Distribution / NonOverflow / MinMSE / + LayerWisePercentile), algorithm selection (CLE / AdaRound / AdaQuant / BiasCorrection / + AutoMixprecision), deployment-target gating (CPU / CUDA / ROCm / AMD + NPU CNN / AMD NPU Transformer), op-type include/exclude lists, weights-only INT4 (MatMulNBits) + decisions, dynamic vs static quantization, or a draft `quant_plan.json`. Trigger for "what + preset should I use for my ONNX model", "choose XINT8 vs A8W8", "plan ONNX PTQ", "INT4 + "BFP16 / MXFP4 for my model", "calibration method for Ryzen AI", + "should I use AdaRound or AdaQuant", "SmoothQuant alpha", or when the user has a model analysis + and needs to decide how to quantize an ONNX model. +layer: l1-atomic +primary_artifact: quant_plan.json +source_knowledge: + - quark/onnx/quantization/config/custom_config.py + - quark/onnx/quantization/config/algorithm.py + - quark/onnx/quantization/config/config.py + - quark/onnx/calibration/methods.py + - examples/onnx/yolo_quantization/quantize_yolo.py + - examples/onnx/accuracy_improvement/quarot/quantize_model.py + - tutorials/onnx/ryzen_ai/resnet50/onnx_ryzen_ai_resnet50_tutorial.ipynb + - tutorials/onnx/ryzen_ai/yolov8/onnx_ryzen_ai_yolov8_tutorial.ipynb + - tutorials/onnx/accuracy_improvement/ + - docs/source/onnx/user_guide_config_description.rst + - docs/source/onnx/appendix_full_quant_config_features.rst +--- + +# quark-onnx-quant-plan + +## Purpose + +Convert an ONNX `model_analysis.json` plus the user's intent into a confirmed `quant_plan.json`. +This skill makes the quantization decisions for the ONNX-to-ONNX flow — which preset, which +calibration method, which algorithm, which op types to include/exclude, whether to enable an NPU +target, whether to use external-data — **without** generating scripts or executing the +quantization. The plan is the contract between the user's intent and the execution step. + +## Inputs + +- `model_analysis.json` from `quark-onnx-model-intake` (architecture, op histogram, opsets, + external-data state, `preset_candidates`, risks) +- `env_context.json` for accelerator-aware preset gating (CUDA major / ROCm major / NPU presence) +- `onnx_install_result.json` (optional) — gates BFP/MX/Extended presets (require the custom-ops + library to be compiled for the target EP) +- User preferences: deployment target, accuracy goal, model-size goal, calibration data + availability + +## Outputs: quant_plan.json + +Records the chosen preset (or custom config), calibration method, algorithm list, layer/op +overrides, NPU flag, external-data setting, and evaluation intent. Shares the schema with the +Torch plan; ONNX-specific fields live under `onnx_specific`. + +Schema: [`quant_plan.schema.json`](../../shared/contracts/quant_plan.schema.json) + +```json +{ + "model": { + "model_type": "onnx", + "architecture_guess": "cnn", + "analysis_ref": "./model_analysis.json" + }, + "backend": "onnx", + "deployment_target": "npu_cnn", + "preset": "XINT8", + "calibration": { + "method": "PowerOfTwo_MinMSE", + "data_size": 200, + "batch_size": 1, + "use_external_data_format": false, + "optimize_mem": false, + "worker_num": 1 + }, + "algorithms": ["CLE"], + "onnx_specific": { + "enable_npu_cnn": true, + "enable_npu_transformer": false, + "include_cle": true, + "include_fast_ft": false, + "op_types_to_quantize": null, + "nodes_to_quantize": null, + "nodes_to_exclude": null, + "subgraphs_to_exclude": [], + "extra_options": { + "OpTypesToExcludeOutputQuantization": [] + }, + "use_external_data_format": false, + "execution_providers": ["CPUExecutionProvider"] + }, + "evaluation_intent": "smoke", + "requires_confirmation": false +} +``` + +## Available Presets (from `DefaultConfigMapping`) + +Quark ONNX ships 50+ named presets in `quark/onnx/quantization/config/custom_config.py`. Pick the +**smallest viable set** for the user's architecture + deployment target; never list them all. + +### AMD NPU CNN — Ryzen AI / VAI (`enable_npu_cnn=True`, PoF2 scales, NHWC) + +| Preset | Description | Picks | +|--------|-------------|-------| +| `XINT8` | INT8 input + INT8 weight, optimized for NPU | Default for any CNN targeting NPU | +| `XINT8_ADAROUND` | XINT8 + AdaRound fast-finetune | When base XINT8 loses accuracy | +| `XINT8_ADAQUANT` | XINT8 + AdaQuant fast-finetune | When AdaRound is not enough | +| `VINT8` | INT8 optimized for VAIML | VAIML deployment | + +### AMD NPU Transformer (`enable_npu_transformer=True`) + +| Preset | Description | Picks | +|--------|-------------|-------| +| `INT16_TRANSFORMER_DEFAULT` | INT16 activations + INT8 weights, fast | Outlier-heavy activations | +| `INT16_TRANSFORMER_ACCURATE` | INT16 + accuracy algorithms | Largest accuracy headroom | + +### General CPU / CUDA / ROCm INT8 (deployment-agnostic) + +| Preset | Description | Picks | +|--------|-------------|-------| +| `A8W8` | INT8 sym activations + INT8 sym weights | Standard CPU/GPU INT8 | +| `A8W8_ADAROUND` / `A8W8_ADAQUANT` | + fast-finetune | Accuracy-tight A8W8 | +| `A16W8` | INT16 sym activations + INT8 sym weights | Wide-activation needs | +| `A16W8_ADAROUND` / `A16W8_ADAQUANT` | + fast-finetune | Accuracy-tight A16W8 | +| `S8S8_AAWS` / `U8S8_AAWS` / `U8U8_AAWA` / `S16S8_ASWS` / `U16S8_AAWS` (+ ADAROUND / ADAQUANT variants) | Various sym/asym INT8/INT16 combos | Fine-tune sym/asym choice per ORT target | +| `INT8_CNN_DEFAULT` / `INT8_CNN_ACCURATE` / `INT16_CNN_DEFAULT` / `INT16_CNN_ACCURATE` | CNN-tuned INT8/INT16 | CPU/GPU CNN deployment | + +### Float Fallbacks + +| Preset | Description | Picks | +|--------|-------------|-------| +| `FP16` / `FP16_ADAQUANT` | FP16 W+A | Accuracy-first when INT is too lossy | +| `BF16` / `BF16_ADAQUANT` | BFloat16 W+A | Same, with BF16 range | + +### Block Formats (require Quark custom-ops library compiled for the target EP) + +| Preset | Description | Picks | +|--------|-------------|-------| +| `BFP16` / `BFP16_ADAQUANT` | Block Floating Point 16-bit | AMD accelerator deployments | +| `MX4` / `MX6` / `MX9` (+ ADAQUANT) | Micro-Exponents BFP variants | Bit-budget exploration | +| `MXFP4E2M1` / `MXFP6E2M3` / `MXFP6E3M2` / `MXFP8E4M3` / `MXFP8E5M2` / `MXINT8` (+ ADAQUANT) | OCP MX formats | Modern AMD/NVIDIA accelerators | + +### Mixed-Precision + +| Preset | Description | Picks | +|--------|-------------|-------| +| `BF16_BFP16` / `BF16_MIXED_BFP16` / `BF16_MIXED_BFP16_ADAQUANT` | BF16 + BFP16 hybrid | High-accuracy + AMD HW | +| `BF16_MXINT8` / `BF16_MIXED_MXINT8` / `BF16_MIXED_MXINT8_ADAQUANT` | BF16 + MXInt8 hybrid | Same, OCP MX flavor | +| `MX9_INT8` | MX9 + INT8 hybrid | Bit-budget exploration | +| `S16S16_MIXED_S8S8` | INT16 + INT8 mixed | Outlier-aware INT mix | + +## Available Calibration Methods + +From `quark/onnx/calibration/methods.py` (+ ORT built-ins): + +| Method | When to pick | +|--------|--------------| +| `MinMax` | Default for most CNN / weights-only; cheap, deterministic | +| `Entropy` | KL-divergence-based; helps when activations have long tails | +| `Percentile` | Clip outliers at a chosen percentile (default 99.999) | +| `Distribution` | Distribution-matching; useful for FP8 `p3`/`same` | +| `LayerWisePercentile` | Auto-picks per-tensor optimal percentile (MAE/MSE) — AMD-specific | +| `PowerOfTwo_NonOverflow` (a.k.a. `NonOverflow`) | **Required for AMD NPU XINT8** — picks the smallest PoF2 scale that doesn't overflow | +| `PowerOfTwo_MinMSE` (a.k.a. `MinMSE`) | **Required for AMD NPU XINT8** — picks the PoF2 scale that minimizes MSE; usually better than NonOverflow | +| `Int16Method.MinMax` | For INT16 configs | + +NPU CNN / XINT8 targets **must** use a `PowerOfTwo*` method. Non-PoF2 scales are rejected at NPU +runtime — flag this as a hard constraint in the plan. + +## Available Algorithms + +From `quark/onnx/quantization/config/algorithm.py` and `examples/onnx/accuracy_improvement/`: + +| Algorithm | Kind | Compatible presets | Description | +|-----------|------|-------------------|-------------| +| `CLE` (Cross-Layer Equalization) | Pre | INT8 CNN configs | Folds BN, equalizes per-channel scales across consecutive Conv/Linear layers (Nagel et al., 2019) | +| `BiasCorrection` | Post | INT8 CNN | Post-hoc bias adjustment (Nagel et al., 2019) | +| `AdaRound` | Post (fast-finetune) | XINT8 / A8W8 / A16W8 / block formats | Adaptive rounding optimization; needs cal data + LR + iterations; GPU-accelerated | +| `AdaQuant` | Post (fast-finetune) | Same as AdaRound | Layer-wise calibration tuning; usually after AdaRound is not enough | +| `AutoMixprecision` | Post | Block formats / mixed-precision | Auto-selects sensitivity-based per-layer dtype; can do dual BFP16+MX hybrid | + +Algorithms compose: e.g. `CLE` (pre) + `AdaRound` (post) is a common XINT8 recipe. Combinations +beyond two algorithms are usually a red flag — flag them in `risks`. + +## Decision Guide + +Help the user choose based on their priorities and the architecture from +`model_analysis.json.model.onnx_specific.architecture_guess`: + +| User intent | Architecture | Recommended starting plan | +|-------------|--------------|---------------------------| +| "Best accuracy, AMD GPU" | any | `BF16` or `BF16_MIXED_BFP16` | +| "INT8 CNN, CPU/GPU deployment" | cnn | `INT8_CNN_DEFAULT` or `A8W8`; add `CLE` if accuracy drops | +| "INT8 CNN → Ryzen AI NPU" | cnn | `XINT8` + `CLE`, calibration = `PowerOfTwo_MinMSE`; usually NHWC pre-conversion via `quark.onnx.tools.convert_nchw_to_nhwc` | +| "Ryzen AI NPU, accuracy-tight CNN" | cnn | `XINT8_ADAROUND` (then `XINT8_ADAQUANT` if still short) | +| "Block format MXFP4 / BFP16 experimentation" | any | `BFP16` or `MXFP4E2M1`; require `quark.onnx.operators.custom_ops` to be compiled | +| "Hybrid mixed-precision for best size/accuracy" | any | `BF16_MIXED_BFP16` or `S16S16_MIXED_S8S8` + `AutoMixprecision` | + +## Deployment-Target Gating (HARD constraints) + +| Target | Preset must satisfy | Calibration must be | Notes | +|--------|--------------------|--------------------|-------| +| `npu_cnn` (Ryzen AI CNN) | `enable_npu_cnn=True`, PoF2 symmetric INT8 per-tensor, NCHW→NHWC done | `PowerOfTwo_MinMSE` or `PowerOfTwo_NonOverflow` | Reject `A8W8` / `BFP16` / `MX*` / `FP16` if user requests `npu_cnn` | +| `npu_transformer` (Ryzen AI Transformer) | `enable_npu_transformer=True`, INT8/INT16 QDQ on MatMul/Gemm | `MinMax` / `Percentile` typically | Reject CNN-only presets | +| `cuda` | `onnxruntime-gpu` present, `CUDAExecutionProvider` available | Any | Block-format presets require custom-ops | +| `rocm` | `onnxruntime_rocm` (ROCm 6.x) **or** CPU ORT on ROCm 7.x (`tools/ci/install_onnxruntime.sh`) | Any | Custom-ops library must compile for ROCm | +| `cpu` | Any | Any | Block-format presets work via CPU custom-ops; expect speed cost | + +If the deployment target conflicts with a requested preset, the plan must either (a) downgrade +to a viable preset and explain, or (b) leave it unset with a high-severity risk in `quant_plan.json`. + +## Op-Type / Node Include/Exclude Levers + +Three commonly-used knobs the plan should expose: + +- **`op_types_to_quantize`** — restrict QDQ insertion to a subset, e.g. `["Conv"]` for + CNN-only quantization. +- **`nodes_to_quantize` / `nodes_to_exclude`** — surgical per-node control by graph node name. + Node names change after pre-processing (NCHW→NHWC, BN folding, etc.), so resolve them after + any pre-processing pass. +- **`extra_options["OpTypesToExcludeOutputQuantization"]`** — keep certain op outputs in float + while still quantizing their inputs/weights. + +## Extra Options Commonly Set in `quant_plan.onnx_specific.extra_options` + +From real examples in `examples/onnx/`: + +| Option | Typical value | Source example | +|--------|---------------|----------------| +| `SimplifyModel` | `True` / `False` | toggle OnnxSlim pre-pass | +| `QuantizeFP16` | `True` | FP16-input models | +| `OpTypesToExcludeOutputQuantization` | `["Add", "Mul"]` etc. | keep selected op outputs in float | +| `FastFinetune` | `{"DataSize": 200, "BatchSize": 2, "NumIterations": 1000, "LearningRate": …, "OptimAlgorithm": "adaround"/"adaquant", "OptimDevice": "cuda:0"/"cpu", "InferDevice": "cuda:0"/"cpu", "EarlyStop": True}` | AdaRound / AdaQuant tutorials and Auto-Search tutorials | + +## Decision Table (MUST show to user) + +**ALWAYS present this table and WAIT for confirmation before finalizing.** Fill the "Value" +column from the user's request, the model analysis, and the deployment-target gates above. + +| Decision | Value | Reason | +|----------|-------|--------| +| `backend` | `onnx` | Fixed for this skill | +| `deployment_target` | _(fill: `cpu` / `cuda` / `rocm` / `npu_cnn` / `npu_transformer`)_ | _(from env or user)_ | +| `preset` | _(fill: name from `DefaultConfigMapping` or `"custom"`)_ | _(why)_ | +| `calibration.method` | _(fill: `MinMax` / `Entropy` / `Percentile` / `Distribution` / `LayerWisePercentile` / `PowerOfTwo_MinMSE` / `PowerOfTwo_NonOverflow`)_ | _(why)_ | +| `calibration.data_size` | `200` (default) | _(why)_ | +| `calibration.batch_size` | `1`–`4` | _(why)_ | +| `algorithms` | _(fill: list e.g. `["CLE"]`, `["AdaRound"]`, `["AdaQuant"]`, `["BiasCorrection"]`)_ | _(why)_ | +| `onnx_specific.enable_npu_cnn` | _(fill: bool)_ | Hard-tied to `deployment_target == "npu_cnn"` | +| `onnx_specific.enable_npu_transformer` | _(fill: bool)_ | Hard-tied to `deployment_target == "npu_transformer"` | +| `onnx_specific.include_cle` | _(fill: bool)_ | CNN INT8 default `true` | +| `onnx_specific.op_types_to_quantize` | `null` or `["MatMul"]` etc. | _(why)_ | +| `onnx_specific.use_external_data_format` | `true` if model > 2 GB (from `model_analysis.json`) | Required for large models | +| `onnx_specific.extra_options` | _(fill: dict)_ | _(why — list each key)_ | +| `evaluation_intent` | `smoke` (default), `mlperf`, `mAP` | _(why)_ | + +After showing the table, ask: **"Confirm this plan? Any changes?"** + +**Do NOT proceed until the user confirms.** + +## Per-Layer Overrides + +For fine-grained control, individual layers can override the global config via `QLayerConfig` +(see `examples/onnx/yolo_quantization/quantize_yolo.py`): + +```python +from quark.onnx import QConfig, QLayerConfig, XInt8Spec, CLEConfig + +config = QConfig( + global_config=QLayerConfig(activation=XInt8Spec(), weight=XInt8Spec()), + algo_config=[CLEConfig()], + EnableNPUCnn=True, + exclude=[ + # YOLOX-style: keep a specific subgraph in float + (["/_head/_modules_list.14/Transpose"], ["/_head/_modules_list.14/Concat_9"]), + ], +) +``` + +Record any per-layer overrides under `onnx_specific.subgraphs_to_exclude` (list of +`(start_nodes, end_nodes)` tuples) or `onnx_specific.nodes_to_exclude` (flat name list). + +## Rules + +- **Keep scope to plan creation only.** Do not generate scripts, do not run quantization, do not + export. Those are separate skills. +- **Require a model analysis first.** Without architecture, op histogram, opsets, and the + already-QDQ flags, you cannot make informed preset / op-type recommendations. If + `model_analysis.json` is missing, route back to `quark-onnx-model-intake`. +- **Honor hard deployment-target constraints.** NPU CNN ⇒ `XINT8` family + `PowerOfTwo*` + calibration + NHWC. NPU Transformer ⇒ `INT8_TRANSFORMER_*` / `INT16_TRANSFORMER_*`. Reject + conflicting preset requests with a clear explanation. +- **Custom-op preconditions for block formats.** Any preset in `{BFP16*, MX*, MXFP*, MXINT8*, + BF16_*BFP*, BF16_*MXINT8*}` requires the Quark custom-ops library to have compiled. If + `onnx_install_result.json` shows the compile failed, suppress these presets from + recommendations and emit a risk pointing back to `quark-onnx-install`. +- **External-data flag tracks model size.** If `model_analysis.json.model.estimated_size_gb` > 2, + set `onnx_specific.use_external_data_format = true` automatically. Otherwise default `false`. +- **Always present the decision table** before finalizing. The user must explicitly confirm. +- **Record risks.** If the user picks a risky combination (e.g. `XINT8` without PoF2 calibration, + `MXFP4` on a model that hasn't been validated with `AutoMixprecision`), keep the user's choice + but record the risk in the plan. +- **Do not duplicate fields between `model_analysis.json` and `quant_plan.json`.** The plan + references the analysis via `analysis_ref`. + +## Interaction Flow + +1. **Check prerequisites**: Is `model_analysis.json` available? If not, route to + `quark-onnx-model-intake` first. Is `onnx_install_result.json` available? If a block-format + preset is on the table, require it. +2. **Confirm deployment target**: From `session_context.json.constraints.deployment_target` or + ask. Apply the hard gates from the table above. +3. **Narrow presets**: Start from `model_analysis.json.quantization_targets.onnx_specific.preset_candidates`, + filter by deployment target and custom-op availability, present the top 1–3 with rationale. +4. **Pick calibration method**: Default per target (MinMax for general; `PowerOfTwo_MinMSE` for + NPU CNN; `Percentile` for outlier-heavy transformers; consider `LayerWisePercentile` when + accuracy is critical). Decide `data_size` and `batch_size`. +5. **Pick algorithms**: Default per architecture and preset (CLE for INT8 CNN; + AdaRound→AdaQuant for accuracy-tight CNN; BiasCorrection as a cheap post-hoc fixup). + At most two algorithms unless justified. +6. **Set extra options**: From the table of common knobs, only set what's needed; document each. +7. **Present the decision table**: Show defaults, explain the tradeoffs, let the user adjust. +8. **Confirm**: Always required. Show the final plan summary before writing. +9. **Emit**: Write `quant_plan.json`. Surface any new constraints back to `the matching public quark-onnx skill` so + they land in `session_context.json`. + +## Recovery + +- If `model_analysis.json` shows `has_qdq_already=true` or `has_quark_custom_ops=true` — do + **not** produce a plan. Route back to `quark-onnx-model-intake` with a recommendation to + remove existing QDQ first (`quark.onnx.tools.remove_qdq`). +- If the user picks a preset that conflicts with their deployment target — explain why it won't + work (e.g. "`A8W8` uses non-PoF2 scales which the AMD NPU CNN runtime rejects"), suggest the + closest viable alternative (e.g. `XINT8`), and respect the user's choice if they insist — + recording the risk. +- If `model_analysis.json` shows architecture `unknown` — produce a draft plan with + `requires_confirmation: true`, list the op histogram, and ask the user which of CNN / + transformer / hybrid path to follow. +- If calibration data is unavailable — explain that PTQ requires representative inputs for + accurate scale estimation, and ask the user to supply a small calibration set (≥ a few dozen + samples) before the plan can be finalized. +- If a block-format preset is requested but custom-ops compile is unverified — hand off to + `quark-onnx-install` to verify, then resume. +- If the user picks an unusual combination (e.g. `CLE` + transformer, three or more algorithms) + — explain why it's atypical and suggest the standard recipe, but respect the user's choice + with a recorded risk. diff --git a/skills/quark-onnx-ptq/skill-card.md b/skills/quark-onnx-ptq/skill-card.md new file mode 100644 index 0000000..cac6dba --- /dev/null +++ b/skills/quark-onnx-ptq/skill-card.md @@ -0,0 +1,13 @@ +# Skill Card + +## Description + +End-to-end ONNX PTQ workflow for AMD Quark — for `.onnx` input models (with optional sibling `.onnx_data` external-weights file). Use when the user wants a complete ONNX-to-ONNX pipeline: model intake, quantization planning, calibration-script generation, manifest, and confirmed execution. + +## Owner + +amd (federated from [amd/Quark](https://github.com/amd/Quark)) + +## License + +MIT diff --git a/skills/quark-onnx-result-validator/.federated.json b/skills/quark-onnx-result-validator/.federated.json new file mode 100644 index 0000000..80d7e20 --- /dev/null +++ b/skills/quark-onnx-result-validator/.federated.json @@ -0,0 +1,10 @@ +{ + "source": "amd-quark", + "repo": "amd/Quark", + "ref": "v0.12", + "commit": "1b229f781a1974cc742884e42d8eefc1eebb4f0a", + "path": ".claude/skills/quark-onnx-result-validator", + "license": "MIT", + "imported_at": "2026-08-05T21:07:39Z", + "implementation_path": ".claude/skills-impl/l1-atomic/onnx/quark-onnx-result-validator" +} diff --git a/skills/quark-onnx-result-validator/SKILL.md b/skills/quark-onnx-result-validator/SKILL.md new file mode 100644 index 0000000..673f086 --- /dev/null +++ b/skills/quark-onnx-result-validator/SKILL.md @@ -0,0 +1,216 @@ +--- +name: quark-onnx-result-validator +description: >- + Validate Quark ONNX quantization output using four lightweight checks: auxiliary file copy + alignment, expected non-quantized initializer MD5 byte-identity (inline `raw_data` + + external-data byte ranges), model metadata equality after stripping quantization-only opset + entries / Quark domains, and fuzzy node-pattern + op-type + dtype summaries with QDQ / + `com.amd.quark` custom-op presence. Intended for post-quantization inspection of `model.onnx` + (with or without `model.onnx_data`). Trigger for "validate ONNX quantization result", "check + quantized .onnx output", "verify ONNX initializers", "did QDQ insertion happen", "are the + non-quantized weights byte-identical". +--- + +# quark-onnx-result-validator + +## Catalog portability + +This standalone skill is federated from `amd/Quark` at commit `1b229f781a1974cc742884e42d8eefc1eebb4f0a`. Resolve bundled files relative to this `SKILL.md`. Repository-relative paths such as `docs/`, `examples/`, and `quark/` refer to the [pinned Quark source](https://github.com/amd/Quark/tree/1b229f781a1974cc742884e42d8eefc1eebb4f0a); use a local Quark checkout when available, otherwise consult that pinned source. + +## Purpose + +Run four lightweight checks on a completed Quark ONNX quantization output. Reads only ONNX graph +headers (`onnx.load(..., load_external_data=False)`), initializer metadata, and small auxiliary +files. Raw payload bytes are only touched for the bounded MD5 spot-check (and only for tensors +matched by the user's exclude rules). Results feed a structured `validation_report.md`. + +## Runtime Assumptions + +All scripts (`quant_validation_onnx.py`, `run_validation.py`) live in the same directory as this +`SKILL.md`, under the installed skill directory. + +Resolve `SKILL_DIR` from the loaded skill location before running any command: + +```bash +# Set SKILL_DIR to the directory containing this installed SKILL.md. +``` + +`run_validation.py` writes JSON to **stdout**; `quant_validation_onnx.py` diagnostics go to +**stderr** with the prefix `[quant-validation-onnx][tag]`. Never treat stderr as structured output. + +Requires the `onnx` Python package in the supported range (`onnx>=1.21.0,<=1.22.0`, per +`requirements.txt`). + +## Contracts + +- Input: `session_context.json`, `quant_plan.json` (for exclude / op-type filters and model paths) +- Output: `validation_report.md` +- Schemas: `shared/contracts/validation_report.schema.json` + +## Inputs + +| Field | Source | Required | +|-------|--------|----------| +| `source_model_path` | user or `session_context` | Step 2 + Step 3 | +| `quantized_model_path` | user or `run_manifest` | All steps | +| `source_model_dir` | parent dir of source model (or `session_context`) | Step 1 | +| `quantized_model_dir` | parent dir of quantized model (or `run_manifest`) | Step 1 | +| `quant_config` | `quant_plan.json` or user-supplied JSON | Step 2 only | + +`quant_config` for step 2 supports the following keys (all optional unless noted): + +| Key | Purpose | Default | +|-----|---------|---------| +| `exclude` | Glob list of initializer names expected to remain unchanged | — | +| `exclude_initializers` | Alias of `exclude` | — | +| `nodes_to_exclude` | Node names whose initializer inputs should remain unchanged | — | +| `op_types_to_quantize` | When set, any initializer **not** wired into one of these op types becomes an implicit exclude | — | +| `max_samples` | Random spot-check cap for large models | `200` | +| `random_seed` / `seed` | Deterministic sampling seed | `None` | + +At least one of `exclude` / `exclude_initializers` / `nodes_to_exclude` / `op_types_to_quantize` +**must** be provided; otherwise step 2 is marked skipped. + +## Outputs + +`validation_report.md` with one section per executed step. Unexecuted steps are marked **skipped**. + +## Interaction Flow + +1. Confirm `SKILL_DIR`, `source_model_path` (if available), and `quantized_model_path` are + resolvable. +2. Run the self-test to verify scripts are intact: + `python3 "$SKILL_DIR/run_validation.py" self-test` +3. Execute steps in cheap-to-expensive order (**4 → 1 → 3 → 2**). +4. Collect JSON from stdout for each step; write `validation_report.md`. +5. Surface any `ok: false` steps with their `errors` / `mismatches`. + +## Four Validation Steps + +| Order | Function | CLI subcommand | Purpose | +|-------|----------|----------------|---------| +| 1 | `check_auxiliary_files_copied` | `auxiliary` | Compare non-`.onnx`/non-`.onnx_data` auxiliary files between source and quantized directories | +| 2 | `check_non_quantized_initializers_md5_unchanged` | `md5` | MD5 spot-check initializer payload bytes (inline `raw_data` or external-data byte ranges) for tensors expected to remain non-quantized | +| 3 | `check_model_metadata_equal_except_quantization` | `metadata` | Compare IR version, producer, default-domain opset, and graph input/output signatures after stripping Quark-injected custom-op domains | +| 4 | `get_fuzzy_node_op_summary` | `fuzzy` | Header-only summary: op-type histogram, canonical node-name patterns, initializer dtype counts per pattern, QDQ / `com.amd.quark` custom-op presence | + +Run in cost order: **4 → 1 → 3 → 2**. + +## Agent Execution Contract + +### Self-Test + +```bash +# Set SKILL_DIR to the directory containing this installed SKILL.md. +python3 "$SKILL_DIR/run_validation.py" self-test +``` + +Exits `0` and prints `exported symbols self-test (__all__): ok` on success. + +### CLI Commands + +```bash +# Set SKILL_DIR to the directory containing this installed SKILL.md. + +# 4. get_fuzzy_node_op_summary (cheapest — header only) +python3 "$SKILL_DIR/run_validation.py" fuzzy \ + --model-path ./quantized/model.onnx + +# 1. check_auxiliary_files_copied +python3 "$SKILL_DIR/run_validation.py" auxiliary \ + --source-model-dir ./source-dir --quantized-model-dir ./quantized-dir \ + --ignore 'README*' + +# 3. check_model_metadata_equal_except_quantization +python3 "$SKILL_DIR/run_validation.py" metadata \ + --source-model-path ./source/model.onnx \ + --quantized-model-path ./quantized/model.onnx + +# 2. check_non_quantized_initializers_md5_unchanged (most expensive) +python3 "$SKILL_DIR/run_validation.py" md5 \ + --source-model-path ./source/model.onnx \ + --output-model-path ./quantized/model.onnx \ + --quant-config '{"exclude":["*.bias","embeddings.*.weight"],"max_samples":50}' +``` + +For `md5`, `--quant-config` accepts a JSON string or a path to a JSON file. +If `SKILL_DIR` or model paths cannot be resolved, mark the affected step **skipped**. + +## Public API (for direct Python import) + +```python +from quant_validation_onnx import ( + check_auxiliary_files_copied, + check_non_quantized_initializers_md5_unchanged, + check_model_metadata_equal_except_quantization, + get_fuzzy_node_op_summary, +) +``` + +All four functions are in `__all__`. Other public-named helpers are internal utility surface. + +## Recovery + +| Failure | Recovery | +|---------|----------| +| Self-test exits non-zero | Report script integrity failure; do not run further steps | +| `onnx` import fails | Hand off to `quark-onnx-install`; do not run any step | +| `source_model_path` missing | Mark steps 2, 3 as skipped; run steps 1, 4 if quantized path is available | +| `quant_config` missing exclude rules | Mark step 2 as skipped | +| External-data file missing alongside `.onnx` | Recorded under `external_data_missing`; affected tensors marked `read_error` in step 2; step 4 still runs against the graph proto | +| Quantized model has zero QDQ / Quark-custom nodes | Step 4 emits a high-severity warning (`quantization_did_not_run`) | + +## Report Template + +```text +## Validation Report — quark-onnx-result-validator + +**Step 4 — fuzzy node / op summary**: ok / FAIL / skipped + - op types: , QDQ nodes: , com.amd.quark nodes: + - Notable: → + - Quantization signal: present / **MISSING** / partial + +**Step 1 — auxiliary files**: ok / FAIL / skipped + - missing: , mismatched: , extra: + +**Step 3 — model metadata**: ok / FAIL / skipped + - ir_version: / + - opset_import (default domain): / + - input/output signature diffs: + - Quark-injected opset domains: + +**Step 2 — MD5 spot-check (initializers)**: ok / FAIL / skipped + - candidates: , checked: , sampled: true/false + - mismatches: + - external_data_missing: +``` + +## Canonical Name Rules + +- Replace only **pure numeric path segments** with `*`: `Conv_12` → `Conv_*`, `model.layer.3.Conv` → + `model.layer.*.Conv` +- Do not alter digits embedded in non-numeric names: `Conv1`, `MatMul_w2` stay unchanged +- `op_type_counts` and `dtype_counts` are aggregated per pattern; multiple op types / dtypes in one + pattern signals partial or mixed-precision quantization + +## Optional Dtype Hints + +Heuristics only — not mandatory pass/fail rules: + +- **INT8 QDQ**: initializers with `INT8` / `UINT8`; nodes `QuantizeLinear` / `DequantizeLinear` +- **INT4 MatMulNBits**: initializers with `UINT8` packed-low-nibble shape; `MatMulNBits` op +- **BFP16 / MX / MXFP**: `com.amd.quark` opset domain present; nodes + `BFPQuantizeDequantize` / `MXQuantizeDequantize` / `ExtendedQuantizeLinear` +- **FP16 / BF16 keep**: initializers with `FLOAT16` / `BFLOAT16` and no QDQ neighbors + +## ONNX-vs-Torch Behavioural Notes + +- Step 2 walks **initializers** (the ONNX analog of safetensors tensors), not safetensors entries. + Inline tensors are read via `tensor.raw_data`; external-data tensors are read by `(offset, + length)` from the external-data file declared in `tensor.external_data`. +- Step 3 compares model-level metadata (IR / producer / opset) and graph I/O signature. ONNX has + no `config.json` equivalent — opset comparison strips Quark's custom domains before equality so + the only allowed diff is the addition of `com.amd.quark` (or similar) on the quantized side. +- Step 4 is structurally similar to the Torch fuzzy summary: it groups nodes / initializers by + canonical name pattern and reports `op_type_counts` / `dtype_counts`. Additionally surfaces + whether quantization actually ran (`QuantizeLinear` / `com.amd.quark` presence). diff --git a/skills/quark-onnx-result-validator/quant_validation_onnx.py b/skills/quark-onnx-result-validator/quant_validation_onnx.py new file mode 100644 index 0000000..32893f6 --- /dev/null +++ b/skills/quark-onnx-result-validator/quant_validation_onnx.py @@ -0,0 +1,1134 @@ +# +# Copyright (C) 2026, Advanced Micro Devices, Inc. All rights reserved. +# SPDX-License-Identifier: MIT +# +# Source file for Claude skill ``quark-onnx-result-validator``. +# Lives at +# ``.claude/skills-impl/l1-atomic/onnx/quark-onnx-result-validator/quant_validation_onnx.py``. + +"""Header-only quantization result validation helpers for Quark ONNX outputs. + +This module performs lightweight checks for ONNX quantization output. Most +logic parses only the ONNX model graph (``onnx.load(..., load_external_data=False)``) +and regular file metadata without decoding full initializer payloads. + +**Agent diagnostics**: the four ``__all__`` entry points emit line-oriented +progress and failure clues to **stderr** using ``[quant-validation-onnx][tag]``. +Machine-consumable fields such as ``ok`` and ``errors`` remain in return values +so diagnostics are not mixed into JSON output. + +Exception: ``check_non_quantized_initializers_md5_unchanged`` reads raw bytes +for initializers matched by exclude rules and calculates MD5 values to confirm +that non-quantized weights are byte-identical. Use ``quant_config["max_samples"]`` +for large models. + +Also, ``check_model_metadata_equal_except_quantization`` reads two ``.onnx`` +models and compares IR version, producer, default-domain opset, and graph +input / output signatures after stripping Quark-injected custom-op domains. +""" + +from __future__ import annotations + +import fnmatch +import hashlib +import random +import re +import sys +import tempfile +from collections import Counter +from pathlib import Path +from typing import Any + +import onnx +from onnx import TensorProto, external_data_helper + +INDEX_SEGMENT = re.compile(r"(?:(?<=^)|(?<=[._/]))\d+(?=[._/]|$)") + +# Files that are weights / payload artifacts. Skip them in auxiliary asset +# alignment, since quantization rewrites them and exact-match content checks are +# not meaningful. +AUXILIARY_IGNORE_PATTERNS = [ + "*.onnx", + "*.onnx_data", + "*.onnx.data", + "*_data", # generic external-data shard naming + "*.pb", + "*.bin", + "*.ckpt", + "*.gguf", + "*.safetensors", + "*.msgpack", + "*.pt", + "*.pth", + "model.onnx.data", +] +AUXILIARY_IGNORE_DIRS = {".git", "__pycache__", ".ipynb_checkpoints", ".cache", ".venv", "node_modules"} + +# Opset domains that Quark may inject as part of quantization. When comparing +# ``opset_import`` across source / quantized, these are stripped from the +# quantized side before equality. +QUARK_INJECTED_DOMAINS: frozenset[str] = frozenset( + { + "com.amd.quark", + "com.microsoft", + "com.microsoft.experimental", + } +) + +# Quark custom-op type names (registered under ``com.amd.quark``). Their +# presence in the quantized graph is a positive signal that quantization ran. +QUARK_CUSTOM_OP_TYPES: frozenset[str] = frozenset( + { + "BFPQuantizeDequantize", + "MXQuantizeDequantize", + "ExtendedQuantizeLinear", + "ExtendedDequantizeLinear", + "ExtendedInstanceNormalization", + "ExtendedLSTM", + } +) + +# Standard ONNX QDQ op types. Their presence is the most common positive signal +# that quantization ran. +QDQ_OP_TYPES: frozenset[str] = frozenset({"QuantizeLinear", "DequantizeLinear", "MatMulNBits"}) + +__all__ = [ + # Auxiliary assets: after ignoring weights / payloads, compare non-weight + # file sets and content on both sides. + "check_auxiliary_files_copied", + # Expected non-quantized initializers: use excludes from ``quant_config`` + # and spot-check raw payload bytes with MD5. + "check_non_quantized_initializers_md5_unchanged", + # Two .onnx models: compare IR / producer / opset (after stripping Quark + # domains) plus graph input/output signature. + "check_model_metadata_equal_except_quantization", + # Header-only summary: op-type histogram, canonical node-name patterns, + # initializer dtype counts, QDQ / custom-op presence. + "get_fuzzy_node_op_summary", +] + + +# --------------------------------------------------------------------------- # +# Common helpers +# --------------------------------------------------------------------------- # + + +def canonicalize_name(name: str) -> str: + """ + Replace pure numeric path segments with ``*``. + + Many ONNX nodes / initializers differ only by numeric indexes + (``Conv_12``, ``model.layer.3.Conv``). Collapsing those into ``*`` + deduplicates them into a compact structural view. + + :param str name: Dotted, underscored, or slashed node / initializer name. + + :return: Canonicalized name. + :rtype: str + """ + return INDEX_SEGMENT.sub("*", name) + + +def _path(path: str | Path) -> Path: + return Path(path).expanduser().resolve() + + +def _agent_log(tag: str, message: str) -> None: + """Write agent-readable progress and exception clues to stderr.""" + print(f"[quant-validation-onnx][{tag}] {message}", file=sys.stderr, flush=True) + + +def _sha256_file(path: Path, max_hash_bytes: int) -> str | None: + if path.stat().st_size > max_hash_bytes: + return None + hasher = hashlib.sha256() + with path.open("rb") as handle: + for block in iter(lambda: handle.read(1024 * 1024), b""): + hasher.update(block) + return hasher.hexdigest() + + +def _matches_any(name: str, patterns: list[str]) -> bool: + return any( + fnmatch.fnmatch(name, pattern) or fnmatch.fnmatch(name.rsplit(".", 1)[0], pattern) for pattern in patterns + ) + + +def _matches_path_pattern(relative_path: Path, patterns: list[str]) -> bool: + path = relative_path.as_posix() + name = relative_path.name + return any(fnmatch.fnmatch(path, pattern) or fnmatch.fnmatch(name, pattern) for pattern in patterns) + + +def _is_auxiliary_file(relative_path: Path, ignore_patterns: list[str]) -> bool: + if any(part in AUXILIARY_IGNORE_DIRS for part in relative_path.parts): + return False + return not _matches_path_pattern(relative_path, ignore_patterns) + + +def _collect_auxiliary_files(model_dir: Path, ignore_patterns: list[str]) -> list[Path]: + files = [] + for path in sorted(model_dir.rglob("*")): + if not path.is_file(): + continue + rel = path.relative_to(model_dir) + if _is_auxiliary_file(rel, ignore_patterns): + files.append(rel) + return files + + +def _dtype_name(dtype_enum: int) -> str: + """Map ``TensorProto.DataType`` enum int to its string name.""" + name_map = {v: k for k, v in TensorProto.DataType.items()} + return name_map.get(dtype_enum, f"UNKNOWN({dtype_enum})") + + +def load_model_header(model_path: str | Path) -> onnx.ModelProto: + """ + Load an ONNX model **without** following external-data references. + + This keeps the parse cheap even for multi-GB models — the graph proto is + parsed in full but tensor payloads stored externally are left on disk. + + :param str | pathlib.Path model_path: Path to the ``.onnx`` file. + + :return: Parsed model proto (initializer payloads may be stubs). + :rtype: onnx.ModelProto + """ + return onnx.load(str(_path(model_path)), load_external_data=False) + + +def collect_initializer_metadata(model: onnx.ModelProto) -> dict[str, dict[str, Any]]: + """ + Collect metadata for every initializer in the model graph. + + :param onnx.ModelProto model: Parsed model (with ``load_external_data=False``). + + :return: Initializer metadata keyed by tensor name. + :rtype: dict[str, dict[str, Any]] + """ + out: dict[str, dict[str, Any]] = {} + for init in model.graph.initializer: + info = { + "dtype": _dtype_name(init.data_type), + "shape": list(init.dims), + "uses_external_data": external_data_helper.uses_external_data(init), + "external_data": {}, + "inline_nbytes": 0, + } + if info["uses_external_data"]: + ed = {entry.key: entry.value for entry in init.external_data} + info["external_data"] = { + "location": ed.get("location"), + "offset": int(ed["offset"]) if "offset" in ed else 0, + "length": int(ed["length"]) if "length" in ed else 0, + } + else: + info["inline_nbytes"] = len(init.raw_data) if init.raw_data else 0 + out[init.name] = info + return out + + +# --------------------------------------------------------------------------- # +# Step 1: auxiliary file alignment +# --------------------------------------------------------------------------- # + + +def check_auxiliary_files_copied( + source_model_dir: str | Path, + quantized_model_dir: str | Path, + ignore: list[str] | None = None, + max_examples: int = 50, + max_hash_bytes: int = 256 * 1024 * 1024, +) -> dict[str, Any]: + """ + Check whether the quantized output copied auxiliary assets from the source model directory. + + Quantization rewrites ``.onnx`` / ``.onnx_data`` files. Non-weight auxiliary files + (``README``, ``LICENSE``, tokenizer / preprocessing JSON, calibration scripts, ``assets/``) + should appear unchanged in the output directory. After applying the default ignore list, + this function compares the remaining path sets and file content (SHA256 for small files, + size only for large ones). + + :param str | pathlib.Path source_model_dir: Source model root directory. + :param str | pathlib.Path quantized_model_dir: Quantized model output root directory. + :param list[str] | None ignore: Additional relative path or file-name glob patterns to + ignore. Merged with ``AUXILIARY_IGNORE_PATTERNS``. + :param int max_examples: Maximum examples returned for list-like fields. + :param int max_hash_bytes: Calculate SHA256 only for files at or below this size. + + :return: Auxiliary file alignment report. + :rtype: dict[str, Any] + """ + _agent_log("aux", "start: check_auxiliary_files_copied (ignoring .onnx/.onnx_data/payloads)") + source_path = _path(source_model_dir) + output_path = _path(quantized_model_dir) + _agent_log("aux", f"source directory: {source_path}") + _agent_log("aux", f"quantized output directory: {output_path}") + ignore_patterns = AUXILIARY_IGNORE_PATTERNS + (ignore or []) + _agent_log("aux", f"merged ignore pattern count: {len(ignore_patterns)}") + errors: list[str] = [] + if not source_path.is_dir(): + errors.append(f"Source model directory not found: {source_path}") + if not output_path.is_dir(): + errors.append(f"Quantized model directory not found: {output_path}") + if errors: + for err in errors: + _agent_log("aux", f"failure: {err}") + _agent_log("aux", "end: check_auxiliary_files_copied (directory unavailable)") + return { + "ok": False, + "source_model_dir": str(source_path), + "quantized_model_dir": str(output_path), + "ignored_patterns": ignore_patterns, + "source_auxiliary_file_count": 0, + "output_auxiliary_file_count": 0, + "missing_file_count": 0, + "mismatched_file_count": 0, + "extra_file_count": 0, + "source_auxiliary_dirs": [], + "missing_dirs": [], + "missing_files": [], + "mismatched_files": [], + "extra_files": [], + "hash_skipped_files": [], + "errors": errors, + } + + source_files = _collect_auxiliary_files(source_path, ignore_patterns) + output_files = _collect_auxiliary_files(output_path, ignore_patterns) + _agent_log( + "aux", + f"source auxiliary file count: {len(source_files)}, output auxiliary file count: {len(output_files)}", + ) + source_set = set(source_files) + output_set = set(output_files) + + missing_files = sorted(source_set - output_set, key=lambda item: item.as_posix()) + extra_files = sorted(output_set - source_set, key=lambda item: item.as_posix()) + _agent_log( + "aux", + f"path set diff: output missing {len(missing_files)}, output extra {len(extra_files)}", + ) + + mismatched_files: list[dict[str, Any]] = [] + hash_skipped_files: list[str] = [] + + for rel in sorted(source_set & output_set, key=lambda item: item.as_posix()): + source_file = source_path / rel + output_file = output_path / rel + source_size = source_file.stat().st_size + output_size = output_file.stat().st_size + if source_size != output_size: + mismatched_files.append( + { + "path": rel.as_posix(), + "source_size": source_size, + "output_size": output_size, + "reason": "size_mismatch", + } + ) + continue + source_hash = _sha256_file(source_file, max_hash_bytes) + output_hash = _sha256_file(output_file, max_hash_bytes) + if source_hash is None or output_hash is None: + hash_skipped_files.append(rel.as_posix()) + continue + if source_hash != output_hash: + mismatched_files.append( + { + "path": rel.as_posix(), + "source_sha256": source_hash, + "output_sha256": output_hash, + "reason": "hash_mismatch", + } + ) + + source_dirs = sorted({p.parent.as_posix() for p in source_files if p.parent.as_posix() != "."}) + missing_dirs = sorted({p.parent.as_posix() for p in missing_files if p.parent.as_posix() != "."}) + + ok = not missing_files and not mismatched_files + _agent_log( + "aux", + f"result: ok={ok} missing={len(missing_files)} mismatched={len(mismatched_files)} " + f"extra={len(extra_files)} hash_skipped={len(hash_skipped_files)}", + ) + _agent_log("aux", "end: check_auxiliary_files_copied") + return { + "ok": ok, + "source_model_dir": str(source_path), + "quantized_model_dir": str(output_path), + "ignored_patterns": ignore_patterns, + "source_auxiliary_file_count": len(source_files), + "output_auxiliary_file_count": len(output_files), + "missing_file_count": len(missing_files), + "mismatched_file_count": len(mismatched_files), + "extra_file_count": len(extra_files), + "source_auxiliary_dirs": source_dirs[:max_examples], + "missing_dirs": missing_dirs[:max_examples], + "missing_files": [p.as_posix() for p in missing_files[:max_examples]], + "mismatched_files": mismatched_files[:max_examples], + "extra_files": [p.as_posix() for p in extra_files[:max_examples]], + "hash_skipped_files": hash_skipped_files[:max_examples], + "errors": [], + } + + +# --------------------------------------------------------------------------- # +# Step 2: non-quantized initializer MD5 spot-check +# --------------------------------------------------------------------------- # + + +def _normalize_exclude_patterns(quant_config: dict[str, Any]) -> list[str]: + """Parse exclude / exclude_initializers / nodes_to_exclude glob lists.""" + + def _to_list(raw: Any) -> list[str]: + if raw is None: + return [] + if isinstance(raw, str): + return [raw] + if isinstance(raw, (list, tuple)): # noqa: UP038 + return [str(item) for item in raw] + return [str(raw)] + + parts = ( + _to_list(quant_config.get("exclude")) + + _to_list(quant_config.get("exclude_initializers")) + + _to_list(quant_config.get("nodes_to_exclude")) + ) + seen: set[str] = set() + merged: list[str] = [] + for item in parts: + if item not in seen: + seen.add(item) + merged.append(item) + return merged + + +def _read_initializer_bytes( + model_path: Path, + init_meta: dict[str, Any], + init_name: str, + raw_inline_lookup: dict[str, bytes], +) -> bytes: + """ + Read the raw payload bytes for a single initializer. + + Inline initializers come from ``raw_inline_lookup`` (a ``{name: raw_data}`` + map built once per model). External-data initializers are seeked + read + from the file declared in ``init_meta['external_data']['location']``, + resolved relative to ``model_path.parent``. + """ + if init_meta.get("uses_external_data"): + ed = init_meta["external_data"] + location = ed.get("location") + if not location: + raise ValueError(f"External-data tensor {init_name!r} has no 'location' entry") + ed_path = (model_path.parent / location).resolve() + if not ed_path.is_file(): + raise FileNotFoundError(f"External-data file not found: {ed_path}") + offset = int(ed["offset"]) + length = int(ed["length"]) + with ed_path.open("rb") as handle: + handle.seek(offset) + data = handle.read(length) + if len(data) != length: + raise ValueError( + f"Unexpected EOF reading external data for {init_name!r}: got {len(data)}, expected {length}" + ) + return data + inline = raw_inline_lookup.get(init_name) + if inline is None: + raise KeyError(f"Inline raw_data not found for initializer {init_name!r}") + return inline + + +def _build_inline_lookup(model: onnx.ModelProto) -> dict[str, bytes]: + return { + init.name: bytes(init.raw_data) + for init in model.graph.initializer + if not external_data_helper.uses_external_data(init) and init.raw_data + } + + +def check_non_quantized_initializers_md5_unchanged( + source_model_path: str | Path, + output_model_path: str | Path, + quant_config: dict[str, Any], +) -> dict[str, Any]: + """ + Check initializers expected to remain non-quantized for byte-identical MD5 values. + + Only initializers present on both sides and matched by ``exclude`` / + ``exclude_initializers`` / ``nodes_to_exclude`` glob rules are checked. + The function compares dtype / shape first, then MD5s raw payload bytes + (inline ``raw_data`` or external-data byte ranges). When candidates exceed + ``quant_config['max_samples']`` (default 200), the set is randomly sampled + using ``quant_config['random_seed']`` / ``['seed']`` when provided. + + :param str | pathlib.Path source_model_path: Original pre-quantization ``.onnx``. + :param str | pathlib.Path output_model_path: Quantized output ``.onnx``. + :param dict[str, Any] quant_config: Configuration with at least one of + ``exclude`` / ``exclude_initializers`` / ``nodes_to_exclude`` provided. + + :return: Report with ``ok``, candidate count, checked count, mismatches. + :rtype: dict[str, Any] + """ + _agent_log( + "md5", + "start: check_non_quantized_initializers_md5_unchanged (will read payload bytes for matched initializers)", + ) + source_path = _path(source_model_path) + output_path = _path(output_model_path) + _agent_log("md5", f"source model: {source_path}") + _agent_log("md5", f"output model: {output_path}") + patterns = _normalize_exclude_patterns(quant_config) + _agent_log("md5", f"exclude rules: {patterns if patterns else '(none)'}") + max_samples_raw = quant_config.get("max_samples", 200) + max_samples = int(max_samples_raw) if max_samples_raw is not None else 200 + if max_samples < 1: + _agent_log("md5", "failure: max_samples < 1") + return { + "ok": False, + "errors": ["max_samples must be >= 1 to run MD5 spot checks."], + "candidate_count": 0, + "checked_count": 0, + "sampled": False, + "exclude_patterns": patterns, + "matched_samples": [], + "mismatches": [], + "external_data_missing": [], + } + + seed = quant_config.get("random_seed", quant_config.get("seed")) + + errors: list[str] = [] + if not patterns: + errors.append( + "quant_config did not provide exclude / exclude_initializers / nodes_to_exclude; " + "no expected non-quantized initializers can be matched." + ) + if not source_path.is_file(): + errors.append(f"Source model file not found: {source_path}") + if not output_path.is_file(): + errors.append(f"Output model file not found: {output_path}") + if errors: + for err in errors: + _agent_log("md5", f"failure: {err}") + return { + "ok": False, + "errors": errors, + "candidate_count": 0, + "checked_count": 0, + "sampled": False, + "max_samples": max_samples, + "exclude_patterns": patterns, + "matched_samples": [], + "mismatches": [], + "external_data_missing": [], + } + + _agent_log("md5", "loading model headers (load_external_data=False)...") + source_model = load_model_header(source_path) + output_model = load_model_header(output_path) + source_meta = collect_initializer_metadata(source_model) + output_meta = collect_initializer_metadata(output_model) + source_inline = _build_inline_lookup(source_model) + output_inline = _build_inline_lookup(output_model) + _agent_log("md5", f"source initializers: {len(source_meta)}, output: {len(output_meta)}") + + candidates = sorted(name for name in source_meta if _matches_any(name, patterns)) + _agent_log("md5", f"candidates matched by excludes: {len(candidates)}") + if not candidates: + return { + "ok": False, + "errors": [], + "candidate_count": 0, + "checked_count": 0, + "sampled": False, + "max_samples": max_samples, + "exclude_patterns": patterns, + "matched_samples": [], + "mismatches": [], + "external_data_missing": [], + "note": "No initializers matched exclude rules; verify globs against the source model's initializer names.", + } + + sampled = len(candidates) > max_samples + if sampled: + rng = random.Random(seed) + checked = sorted(rng.sample(candidates, max_samples)) + _agent_log("md5", f"sampled {len(checked)}/{len(candidates)} candidates") + else: + checked = list(candidates) + + matched_samples: list[str] = [] + mismatches: list[dict[str, Any]] = [] + external_data_missing: set[str] = set() + + for step, name in enumerate(checked, start=1): + if name not in output_meta: + mismatches.append({"name": name, "reason": "missing_in_output"}) + _agent_log("md5", f"[{step}/{len(checked)}] FAIL {name}: key missing in output") + continue + s_meta = source_meta[name] + o_meta = output_meta[name] + if s_meta["dtype"] != o_meta["dtype"] or s_meta["shape"] != o_meta["shape"]: + mismatches.append( + { + "name": name, + "reason": "header_mismatch", + "source_dtype": s_meta["dtype"], + "output_dtype": o_meta["dtype"], + "source_shape": s_meta["shape"], + "output_shape": o_meta["shape"], + } + ) + _agent_log( + "md5", + f"[{step}/{len(checked)}] FAIL {name}: header mismatch " + f"{s_meta['dtype']}/{o_meta['dtype']} {s_meta['shape']}/{o_meta['shape']}", + ) + continue + try: + s_bytes = _read_initializer_bytes(source_path, s_meta, name, source_inline) + o_bytes = _read_initializer_bytes(output_path, o_meta, name, output_inline) + except FileNotFoundError as exc: + external_data_missing.add(str(exc).split(": ", 1)[-1]) + mismatches.append({"name": name, "reason": "read_error", "detail": str(exc)}) + _agent_log("md5", f"[{step}/{len(checked)}] FAIL {name}: external-data file missing") + continue + except (OSError, ValueError, KeyError) as exc: + mismatches.append({"name": name, "reason": "read_error", "detail": str(exc)}) + _agent_log("md5", f"[{step}/{len(checked)}] FAIL {name}: payload read error {exc!r}") + continue + s_md5 = hashlib.md5(s_bytes).hexdigest() + o_md5 = hashlib.md5(o_bytes).hexdigest() + if s_md5 != o_md5: + mismatches.append({"name": name, "reason": "md5_mismatch", "source_md5": s_md5, "output_md5": o_md5}) + _agent_log("md5", f"[{step}/{len(checked)}] FAIL {name}: MD5 mismatch") + else: + matched_samples.append(name) + _agent_log("md5", f"[{step}/{len(checked)}] OK {name}: MD5 matched") + + ok = not mismatches + _agent_log( + "md5", + f"result: ok={ok} checked={len(checked)} matched={len(matched_samples)} " + f"mismatches={len(mismatches)} external_data_missing={len(external_data_missing)}", + ) + _agent_log("md5", "end: check_non_quantized_initializers_md5_unchanged") + return { + "ok": ok, + "errors": [], + "candidate_count": len(candidates), + "checked_count": len(checked), + "sampled": sampled, + "max_samples": max_samples, + "random_seed": seed, + "exclude_patterns": patterns, + "matched_samples": matched_samples[:100], + "mismatches": mismatches[:200], + "external_data_missing": sorted(external_data_missing)[:50], + } + + +# --------------------------------------------------------------------------- # +# Step 3: model metadata equality (modulo Quark-injected domains) +# --------------------------------------------------------------------------- # + + +def _summarize_io(value_infos: list[Any]) -> list[dict[str, Any]]: + """Convert a graph input/output proto list into a compact dict list.""" + out = [] + for vi in value_infos: + t = vi.type.tensor_type + shape = [] + for d in t.shape.dim: + if d.HasField("dim_value"): + shape.append(d.dim_value) + elif d.HasField("dim_param"): + shape.append(d.dim_param) + else: + shape.append("?") + out.append({"name": vi.name, "dtype": _dtype_name(t.elem_type), "shape": shape}) + return out + + +def _opsets_excluding(model: onnx.ModelProto, excluded_domains: frozenset[str]) -> list[dict[str, Any]]: + result = [] + for op in model.opset_import: + domain = op.domain or "ai.onnx" + if domain in excluded_domains: + continue + result.append({"domain": domain, "version": op.version}) + return sorted(result, key=lambda x: (x["domain"], x["version"])) + + +def check_model_metadata_equal_except_quantization( + source_model_path: str | Path, + quantized_model_path: str | Path, + extra_excluded_domains: list[str] | None = None, +) -> dict[str, Any]: + """ + Compare model-level metadata after stripping Quark-injected opset domains. + + Checks that ``ir_version``, ``producer_name``, ``producer_version``, + ``model_version``, ``domain``, the default-domain opset version, and the + graph input / output signatures are identical between source and quantized + models. Opset entries for domains in ``QUARK_INJECTED_DOMAINS`` (and any + additional domains supplied via ``extra_excluded_domains``) are stripped + from the quantized side before comparison — quantization is allowed to + *add* a ``com.amd.quark`` opset entry but not to change the default domain. + + :param str | pathlib.Path source_model_path: Original ``.onnx``. + :param str | pathlib.Path quantized_model_path: Quantized ``.onnx``. + :param list[str] | None extra_excluded_domains: Additional domains to strip + from the quantized side before opset equality. + + :return: ``ok`` and a list of ``mismatches`` with field-level detail. + :rtype: dict[str, Any] + """ + _agent_log("metadata", "start: check_model_metadata_equal_except_quantization") + source_path = _path(source_model_path) + quant_path = _path(quantized_model_path) + _agent_log("metadata", f"source model: {source_path}") + _agent_log("metadata", f"quantized model: {quant_path}") + errors: list[str] = [] + if not source_path.is_file(): + errors.append(f"Source model not found: {source_path}") + if not quant_path.is_file(): + errors.append(f"Quantized model not found: {quant_path}") + if errors: + for err in errors: + _agent_log("metadata", f"failure: {err}") + return { + "ok": False, + "errors": errors, + "mismatches": [], + "mismatch_count": 0, + "source_model_path": str(source_path), + "quantized_model_path": str(quant_path), + "quark_injected_domains": [], + } + + excluded = frozenset(QUARK_INJECTED_DOMAINS) | frozenset(extra_excluded_domains or []) + source_model = load_model_header(source_path) + quant_model = load_model_header(quant_path) + + mismatches: list[str] = [] + + for field in ("ir_version", "producer_name", "producer_version", "model_version", "domain"): + s_val = getattr(source_model, field) + q_val = getattr(quant_model, field) + if s_val != q_val: + mismatches.append(f"{field}: {s_val!r} / {q_val!r}") + + s_opsets = _opsets_excluding(source_model, frozenset()) + q_opsets_full = _opsets_excluding(quant_model, frozenset()) + q_opsets_stripped = _opsets_excluding(quant_model, excluded) + quark_added = sorted( + {(op["domain"], op["version"]) for op in q_opsets_full} - {(op["domain"], op["version"]) for op in s_opsets} + ) + + if s_opsets != q_opsets_stripped: + mismatches.append( + f"opset_import (after stripping Quark domains from quantized): {s_opsets} / {q_opsets_stripped}" + ) + + s_inputs = _summarize_io(list(source_model.graph.input)) + q_inputs = _summarize_io(list(quant_model.graph.input)) + if s_inputs != q_inputs: + mismatches.append(f"graph.input: {s_inputs} / {q_inputs}") + + s_outputs = _summarize_io(list(source_model.graph.output)) + q_outputs = _summarize_io(list(quant_model.graph.output)) + if s_outputs != q_outputs: + mismatches.append(f"graph.output: {s_outputs} / {q_outputs}") + + ok = not mismatches + _agent_log( + "metadata", + f"result: ok={ok} mismatch_count={len(mismatches)} quark_added_opsets={quark_added}", + ) + for line in mismatches[:25]: + _agent_log("metadata", f" diff: {line}") + _agent_log("metadata", "end: check_model_metadata_equal_except_quantization") + return { + "ok": ok, + "errors": [], + "mismatches": mismatches[:200], + "mismatch_count": len(mismatches), + "source_model_path": str(source_path), + "quantized_model_path": str(quant_path), + "quark_injected_domains": [{"domain": d, "version": v} for d, v in quark_added], + "stripped_domains": sorted(excluded), + } + + +# --------------------------------------------------------------------------- # +# Step 4: fuzzy node / op / initializer summary +# --------------------------------------------------------------------------- # + + +def get_fuzzy_node_op_summary(model_path: str | Path) -> dict[str, Any]: + """ + Collect a header-only structural summary of a quantized ONNX model. + + Surfaces: + + - **op_type_counts**: full op-type histogram for the graph. + - **node_patterns**: canonical node-name patterns (numeric path segments + collapsed to ``*``) with per-pattern ``op_type_counts``. A pattern that + maps to multiple op types signals partial quantization. + - **initializer_dtype_patterns**: canonical initializer-name patterns with + per-pattern ``dtype_counts``. Mixed dtypes within one pattern usually + indicates partial quantization (e.g. some layers quantized, others not). + - **qdq_op_count**: number of ``QuantizeLinear`` / ``DequantizeLinear`` / + ``MatMulNBits`` nodes. + - **quark_custom_op_count**: number of nodes whose op type is in + ``QUARK_CUSTOM_OP_TYPES`` (BFP / MX / Extended). + - **quark_opset_domains**: subset of ``opset_import`` that overlaps with + ``QUARK_INJECTED_DOMAINS``. + - **quantization_signal**: ``"present"`` / ``"missing"`` / ``"partial"`` + depending on QDQ and custom-op counts. + + :param str | pathlib.Path model_path: Path to an ``.onnx`` file. + + :return: Summary dict (see fields above). + :rtype: dict[str, Any] + """ + _agent_log("fuzzy", "start: get_fuzzy_node_op_summary (header only)") + resolved = _path(model_path) + _agent_log("fuzzy", f"resolved model path -> {resolved}") + if not resolved.is_file(): + _agent_log("fuzzy", f"failure: model not found: {resolved}") + return { + "ok": False, + "errors": [f"Model file not found: {resolved}"], + "model_path": str(resolved), + } + model = load_model_header(resolved) + graph = model.graph + + op_counts: Counter[str] = Counter() + node_pattern_order: list[str] = [] + node_pattern_op_counts: dict[str, Counter[str]] = {} + qdq_count = 0 + quark_custom_count = 0 + + for node in graph.node: + op_counts[node.op_type] += 1 + if node.op_type in QDQ_OP_TYPES: + qdq_count += 1 + if node.op_type in QUARK_CUSTOM_OP_TYPES or node.domain in QUARK_INJECTED_DOMAINS: + quark_custom_count += 1 + node_name = node.name or f"" + pattern = canonicalize_name(node_name) + if pattern not in node_pattern_op_counts: + node_pattern_op_counts[pattern] = Counter() + node_pattern_order.append(pattern) + node_pattern_op_counts[pattern][node.op_type] += 1 + + init_pattern_order: list[str] = [] + init_pattern_dtype_counts: dict[str, Counter[str]] = {} + for init in graph.initializer: + pattern = canonicalize_name(init.name) + if pattern not in init_pattern_dtype_counts: + init_pattern_dtype_counts[pattern] = Counter() + init_pattern_order.append(pattern) + init_pattern_dtype_counts[pattern][_dtype_name(init.data_type)] += 1 + + quark_opset_domains = sorted({op.domain for op in model.opset_import if op.domain in QUARK_INJECTED_DOMAINS}) + + if qdq_count + quark_custom_count == 0: + signal = "missing" + elif qdq_count > 0 and quark_custom_count > 0: + signal = "present" + else: + signal = "present" + + node_patterns = [] + for pattern in node_pattern_order: + counts = node_pattern_op_counts[pattern] + sorted_items = sorted(counts.items(), key=lambda x: (-x[1], x[0])) + node_patterns.append({"pattern": pattern, "op_type_counts": dict(sorted_items)}) + + init_patterns = [] + for pattern in init_pattern_order: + counts = init_pattern_dtype_counts[pattern] + sorted_items = sorted(counts.items(), key=lambda x: (-x[1], x[0])) + init_patterns.append({"pattern": pattern, "dtype_counts": dict(sorted_items)}) + + mixed_node_patterns = sum(1 for item in node_patterns if len(item["op_type_counts"]) > 1) + mixed_init_patterns = sum(1 for item in init_patterns if len(item["dtype_counts"]) > 1) + + _agent_log( + "fuzzy", + f"op types: {len(op_counts)}, node patterns: {len(node_patterns)} (mixed: {mixed_node_patterns}), " + f"init patterns: {len(init_patterns)} (mixed: {mixed_init_patterns})", + ) + _agent_log( + "fuzzy", + f"qdq_op_count={qdq_count}, quark_custom_op_count={quark_custom_count}, " + f"quark_opset_domains={quark_opset_domains}, signal={signal}", + ) + _agent_log("fuzzy", "end: get_fuzzy_node_op_summary") + return { + "ok": True, + "errors": [], + "model_path": str(resolved), + "op_type_counts": dict(sorted(op_counts.items(), key=lambda x: (-x[1], x[0]))), + "qdq_op_count": qdq_count, + "quark_custom_op_count": quark_custom_count, + "quark_opset_domains": quark_opset_domains, + "quantization_signal": signal, + "node_patterns_count": len(node_patterns), + "node_patterns_mixed_op": mixed_node_patterns, + "node_patterns": node_patterns[:300], + "initializer_patterns_count": len(init_patterns), + "initializer_patterns_mixed_dtype": mixed_init_patterns, + "initializer_patterns": init_patterns[:300], + } + + +# --------------------------------------------------------------------------- # +# Mock model builders for the embedded self-test +# --------------------------------------------------------------------------- # + + +def _make_tensor_inline(name: str, dtype: int, dims: list[int], raw: bytes) -> Any: + t = TensorProto() + t.name = name + t.data_type = dtype + t.dims.extend(dims) + t.raw_data = raw + return t + + +def _make_minimal_model( + opset_version: int, + initializers: list[Any], + extra_opset_domain: str | None = None, + extra_opset_version: int = 1, + extra_nodes: list[Any] | None = None, +) -> onnx.ModelProto: + """Build a tiny model with one Identity node so the graph is valid.""" + graph = onnx.GraphProto() + graph.name = "g" + + # one float input/output so the graph is valid + inp = onnx.ValueInfoProto() + inp.name = "in" + inp.type.tensor_type.elem_type = TensorProto.FLOAT + d = inp.type.tensor_type.shape.dim.add() + d.dim_value = 1 + outp = onnx.ValueInfoProto() + outp.name = "out" + outp.type.tensor_type.elem_type = TensorProto.FLOAT + d = outp.type.tensor_type.shape.dim.add() + d.dim_value = 1 + graph.input.append(inp) + graph.output.append(outp) + + n = onnx.NodeProto() + n.op_type = "Identity" + n.name = "Identity_0" + n.input.append("in") + n.output.append("out") + graph.node.append(n) + + for ex in extra_nodes or []: + graph.node.append(ex) + for init in initializers: + graph.initializer.append(init) + + model = onnx.ModelProto() + model.ir_version = 9 + model.producer_name = "quark-validator-selftest" + model.producer_version = "0.1" + model.graph.CopyFrom(graph) + op = model.opset_import.add() + op.domain = "" + op.version = opset_version + if extra_opset_domain is not None: + op2 = model.opset_import.add() + op2.domain = extra_opset_domain + op2.version = extra_opset_version + return model + + +def _selftest_exported_symbols() -> int: + """Run mock-data checks for symbols listed in ``__all__``.""" + failures: list[str] = [] + + with tempfile.TemporaryDirectory() as tmp: + base = Path(tmp) + + # -------- get_fuzzy_node_op_summary ---------------------------------- + fuzzy_dir = base / "fuzzy" + fuzzy_dir.mkdir() + fuzzy_inits = [ + _make_tensor_inline("model.layers.0.weight", TensorProto.INT8, [2], b"\x01\x02"), + _make_tensor_inline("model.layers.1.weight", TensorProto.INT8, [2], b"\x03\x04"), + _make_tensor_inline("model.head.weight", TensorProto.FLOAT, [1], b"\x00\x00\x00\x00"), + ] + # Add a QuantizeLinear + a com.amd.quark BFPQuantizeDequantize node. + qdq_node = onnx.NodeProto() + qdq_node.op_type = "QuantizeLinear" + qdq_node.name = "QuantizeLinear_0" + qdq_node.input.extend(["in", "model.layers.0.weight"]) + qdq_node.output.append("q0") + bfp_node = onnx.NodeProto() + bfp_node.op_type = "BFPQuantizeDequantize" + bfp_node.name = "BFPQuantizeDequantize_0" + bfp_node.domain = "com.amd.quark" + bfp_node.input.append("in") + bfp_node.output.append("bfp0") + fuzzy_model = _make_minimal_model( + opset_version=17, + initializers=fuzzy_inits, + extra_opset_domain="com.amd.quark", + extra_opset_version=1, + extra_nodes=[qdq_node, bfp_node], + ) + fuzzy_path = fuzzy_dir / "model.onnx" + onnx.save(fuzzy_model, str(fuzzy_path)) + fuzzy_summary = get_fuzzy_node_op_summary(fuzzy_path) + if not fuzzy_summary.get("ok"): + failures.append(f"get_fuzzy_node_op_summary: not ok: {fuzzy_summary}") + if fuzzy_summary.get("qdq_op_count") != 1: + failures.append(f"get_fuzzy_node_op_summary: expected 1 qdq, got {fuzzy_summary.get('qdq_op_count')}") + if fuzzy_summary.get("quark_custom_op_count") != 1: + failures.append( + f"get_fuzzy_node_op_summary: expected 1 quark custom op, got {fuzzy_summary.get('quark_custom_op_count')}" + ) + if "com.amd.quark" not in fuzzy_summary.get("quark_opset_domains", []): + failures.append("get_fuzzy_node_op_summary: missing com.amd.quark opset domain") + # initializer pattern canonicalization + init_patterns = { + item["pattern"]: item["dtype_counts"] for item in fuzzy_summary.get("initializer_patterns", []) + } + if "model.layers.*.weight" not in init_patterns: + failures.append(f"get_fuzzy_node_op_summary: expected canonical pattern, got {list(init_patterns)}") + + # -------- check_auxiliary_files_copied ------------------------------- + src_aux = base / "src_aux" + out_aux = base / "out_aux" + src_aux.mkdir() + out_aux.mkdir() + (src_aux / "README.md").write_text("readme", encoding="utf-8") + (out_aux / "README.md").write_text("readme", encoding="utf-8") + # Write differing .onnx payloads — should be IGNORED. + onnx.save(_make_minimal_model(17, []), str(src_aux / "model.onnx")) + onnx.save( + _make_minimal_model(17, [_make_tensor_inline("w", TensorProto.INT8, [1], b"\x01")]), + str(out_aux / "model.onnx"), + ) + aux_rep = check_auxiliary_files_copied(src_aux, out_aux, max_examples=20) + if not aux_rep.get("ok"): + failures.append(f"check_auxiliary_files_copied: expected ok, got {aux_rep}") + + # missing file case + (src_aux / "only_in_source.txt").write_text("only", encoding="utf-8") + aux_rep2 = check_auxiliary_files_copied(src_aux, out_aux, max_examples=20) + if aux_rep2.get("ok"): + failures.append("check_auxiliary_files_copied: expected FAIL when source has extra non-weight file") + + # -------- check_model_metadata_equal_except_quantization ------------- + meta_dir = base / "meta" + meta_dir.mkdir() + src_meta_model = _make_minimal_model(17, []) + # Quantized version adds com.amd.quark to opset (should still be ok). + quant_meta_model = _make_minimal_model(17, [], extra_opset_domain="com.amd.quark") + src_meta_path = meta_dir / "src.onnx" + quant_meta_path = meta_dir / "quant.onnx" + onnx.save(src_meta_model, str(src_meta_path)) + onnx.save(quant_meta_model, str(quant_meta_path)) + meta_rep = check_model_metadata_equal_except_quantization(src_meta_path, quant_meta_path) + if not meta_rep.get("ok"): + failures.append(f"check_model_metadata_equal_except_quantization: expected ok, got {meta_rep}") + if not any(d["domain"] == "com.amd.quark" for d in meta_rep.get("quark_injected_domains", [])): + failures.append( + "check_model_metadata_equal_except_quantization: expected com.amd.quark in quark_injected_domains" + ) + + # Now bump opset on the quantized side — should FAIL. + bad_meta_model = _make_minimal_model(18, [], extra_opset_domain="com.amd.quark") + bad_meta_path = meta_dir / "bad.onnx" + onnx.save(bad_meta_model, str(bad_meta_path)) + meta_bad = check_model_metadata_equal_except_quantization(src_meta_path, bad_meta_path) + if meta_bad.get("ok"): + failures.append("check_model_metadata_equal_except_quantization: expected FAIL on opset bump") + + # -------- check_non_quantized_initializers_md5_unchanged ------------- + md5_dir = base / "md5" + md5_dir.mkdir() + payload = b"keep_payload_bytes_12345678" + src_md5_model = _make_minimal_model( + 17, [_make_tensor_inline("model.layers.0.keep.weight", TensorProto.UINT8, [len(payload)], payload)] + ) + ok_md5_model = _make_minimal_model( + 17, [_make_tensor_inline("model.layers.0.keep.weight", TensorProto.UINT8, [len(payload)], payload)] + ) + bad_md5_model = _make_minimal_model( + 17, [_make_tensor_inline("model.layers.0.keep.weight", TensorProto.UINT8, [len(payload)], payload[::-1])] + ) + src_md5_path = md5_dir / "src.onnx" + ok_md5_path = md5_dir / "ok.onnx" + bad_md5_path = md5_dir / "bad.onnx" + onnx.save(src_md5_model, str(src_md5_path)) + onnx.save(ok_md5_model, str(ok_md5_path)) + onnx.save(bad_md5_model, str(bad_md5_path)) + qconf: dict[str, Any] = {"exclude": ["*.keep.weight"], "max_samples": 20, "random_seed": 0} + md5_ok = check_non_quantized_initializers_md5_unchanged(src_md5_path, ok_md5_path, qconf) + if not md5_ok.get("ok") or "model.layers.0.keep.weight" not in md5_ok.get("matched_samples", []): + failures.append(f"check_non_quantized_initializers_md5_unchanged success case: {md5_ok}") + md5_bad = check_non_quantized_initializers_md5_unchanged(src_md5_path, bad_md5_path, qconf) + if md5_bad.get("ok"): + failures.append("check_non_quantized_initializers_md5_unchanged: expected FAIL on byte change") + reasons = {m.get("reason") for m in md5_bad.get("mismatches", [])} + if "md5_mismatch" not in reasons: + failures.append(f"check_non_quantized_initializers_md5_unchanged: expected md5_mismatch in {md5_bad}") + + # No-exclude case + md5_none = check_non_quantized_initializers_md5_unchanged(src_md5_path, ok_md5_path, {}) + if md5_none.get("ok"): + failures.append("check_non_quantized_initializers_md5_unchanged: expected FAIL when no excludes given") + + # Sampling branch + samp_inits_src = [ + _make_tensor_inline(f"model.layers.{i}.keep.weight", TensorProto.UINT8, [2], b"\xaa\xbb") for i in range(8) + ] + samp_inits_out = [ + _make_tensor_inline(f"model.layers.{i}.keep.weight", TensorProto.UINT8, [2], b"\xaa\xbb") for i in range(8) + ] + src_samp_path = md5_dir / "src_samp.onnx" + out_samp_path = md5_dir / "out_samp.onnx" + onnx.save(_make_minimal_model(17, samp_inits_src), str(src_samp_path)) + onnx.save(_make_minimal_model(17, samp_inits_out), str(out_samp_path)) + qconf_samp = {"exclude": ["*.keep.weight"], "max_samples": 3, "random_seed": 42} + md5_samp = check_non_quantized_initializers_md5_unchanged(src_samp_path, out_samp_path, qconf_samp) + if not md5_samp.get("ok"): + failures.append(f"check_non_quantized_initializers_md5_unchanged sampling: {md5_samp}") + if not md5_samp.get("sampled") or md5_samp.get("checked_count") != 3: + failures.append( + f"check_non_quantized_initializers_md5_unchanged: expected sampled=True checked=3, got {md5_samp}" + ) + + if failures: + for line in failures: + print(line, file=sys.stderr) + return 1 + + print("exported symbols self-test (__all__): ok") + return 0 + + +def run_selftest() -> int: + """Public entry point for the embedded self-test. Returns 0 on success, 1 on failure.""" + return _selftest_exported_symbols() + + +if __name__ == "__main__": + raise SystemExit(_selftest_exported_symbols()) diff --git a/skills/quark-onnx-result-validator/run_validation.py b/skills/quark-onnx-result-validator/run_validation.py new file mode 100644 index 0000000..8d3eef5 --- /dev/null +++ b/skills/quark-onnx-result-validator/run_validation.py @@ -0,0 +1,138 @@ +# +# Copyright (C) 2026, Advanced Micro Devices, Inc. All rights reserved. +# SPDX-License-Identifier: MIT +# + +"""CLI for the vendored ``quant_validation_onnx`` module in this skill directory. + +Resolves imports without ``PYTHONPATH`` by prepending this file's directory to ``sys.path``. +""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +_SKILL_DIR = Path(__file__).resolve().parent +if str(_SKILL_DIR) not in sys.path: + sys.path.insert(0, str(_SKILL_DIR)) + +import quant_validation_onnx as qv # noqa: E402 + + +def _print_json(data: object) -> None: + print(json.dumps(data, ensure_ascii=False, indent=2)) + + +def _load_json_arg(raw: str) -> dict: + path = Path(raw).expanduser() + if path.is_file(): + with path.open("r", encoding="utf-8") as handle: + return json.load(handle) + return json.loads(raw) + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Quark ONNX quant output checks (skill vendored quant_validation_onnx)." + ) + sub = parser.add_subparsers(dest="command", required=True) + + p_self = sub.add_parser("self-test", help="Run embedded mock-data checks for quant_validation_onnx.__all__.") + p_self.set_defaults(func=_cmd_self_test) + + p_fuzzy = sub.add_parser( + "fuzzy", + help="Summarize op-type histogram, canonical node patterns, initializer dtype counts, " + "and QDQ / com.amd.quark presence (header-only).", + ) + p_fuzzy.add_argument("--model-path", required=True, help="Path to a single .onnx file.") + p_fuzzy.set_defaults(func=_cmd_fuzzy) + + p_aux = sub.add_parser("auxiliary", help="Compare non-weight auxiliary files between two model directories.") + p_aux.add_argument("--source-model-dir", required=True) + p_aux.add_argument("--quantized-model-dir", required=True) + p_aux.add_argument("--ignore", nargs="*", default=[]) + p_aux.add_argument("--max-examples", type=int, default=50) + p_aux.add_argument("--max-hash-bytes", type=int, default=256 * 1024 * 1024) + p_aux.set_defaults(func=_cmd_auxiliary) + + p_meta = sub.add_parser( + "metadata", + help="Compare two .onnx models' IR / producer / opset / graph I/O signature after " + "stripping Quark-injected opset domains.", + ) + p_meta.add_argument("--source-model-path", required=True) + p_meta.add_argument("--quantized-model-path", required=True) + p_meta.add_argument( + "--extra-excluded-domain", + action="append", + default=None, + help="Extra opset domain to strip from the quantized side before equality (may be repeated).", + ) + p_meta.set_defaults(func=_cmd_metadata) + + p_md5 = sub.add_parser( + "md5", + help="MD5 spot-check exclude-listed initializers (reads raw_data + external-data byte ranges).", + ) + p_md5.add_argument("--source-model-path", required=True) + p_md5.add_argument("--output-model-path", required=True) + p_md5.add_argument( + "--quant-config", + required=True, + help="JSON object string or path to JSON (exclude / exclude_initializers / nodes_to_exclude, " + "max_samples, random_seed, ...).", + ) + p_md5.set_defaults(func=_cmd_md5) + + args = parser.parse_args() + args.func(args) + + +def _cmd_self_test(_args: argparse.Namespace) -> None: + raise SystemExit(qv.run_selftest()) + + +def _cmd_fuzzy(args: argparse.Namespace) -> None: + out = qv.get_fuzzy_node_op_summary(args.model_path) + _print_json(out) + + +def _cmd_auxiliary(args: argparse.Namespace) -> None: + out = qv.check_auxiliary_files_copied( + args.source_model_dir, + args.quantized_model_dir, + ignore=args.ignore or None, + max_examples=args.max_examples, + max_hash_bytes=args.max_hash_bytes, + ) + _print_json(out) + + +def _cmd_metadata(args: argparse.Namespace) -> None: + out = qv.check_model_metadata_equal_except_quantization( + args.source_model_path, + args.quantized_model_path, + extra_excluded_domains=args.extra_excluded_domain, + ) + _print_json(out) + + +def _cmd_md5(args: argparse.Namespace) -> None: + qconf = _load_json_arg(args.quant_config) + if not isinstance(qconf, dict): + print("quant-config must be a JSON object", file=sys.stderr) + raise SystemExit(2) + out = qv.check_non_quantized_initializers_md5_unchanged( + args.source_model_path, + args.output_model_path, + qconf, + ) + _print_json(out) + + +if __name__ == "__main__": + main() diff --git a/skills/quark-onnx-result-validator/skill-card.md b/skills/quark-onnx-result-validator/skill-card.md new file mode 100644 index 0000000..9534a71 --- /dev/null +++ b/skills/quark-onnx-result-validator/skill-card.md @@ -0,0 +1,13 @@ +# Skill Card + +## Description + +Validate Quark ONNX quantization output using four lightweight checks: auxiliary file copy alignment, expected non-quantized initializer MD5 byte-identity (inline `raw_data` + external-data byte ranges), model metadata equality after stripping quantization-only opset entries / Quark domains, and fuzzy node-pattern + o… + +## Owner + +amd (federated from [amd/Quark](https://github.com/amd/Quark)) + +## License + +MIT diff --git a/skills/quark-torch-debug/.federated.json b/skills/quark-torch-debug/.federated.json new file mode 100644 index 0000000..e0f1077 --- /dev/null +++ b/skills/quark-torch-debug/.federated.json @@ -0,0 +1,10 @@ +{ + "source": "amd-quark", + "repo": "amd/Quark", + "ref": "v0.12", + "commit": "1b229f781a1974cc742884e42d8eefc1eebb4f0a", + "path": ".claude/skills/quark-torch-debug", + "license": "MIT", + "imported_at": "2026-08-05T21:07:39Z", + "implementation_path": ".claude/skills-impl/l1-atomic/torch/quark-torch-debug" +} diff --git a/skills/quark-torch-debug/SKILL.md b/skills/quark-torch-debug/SKILL.md new file mode 100644 index 0000000..52a58ae --- /dev/null +++ b/skills/quark-torch-debug/SKILL.md @@ -0,0 +1,172 @@ +--- +name: quark-torch-debug +description: >- + Diagnose failed Quark Torch PTQ installation, execution, script generation, or export attempts. + Use when the user reports a torch-side error, stack trace, invalid artifact, missing dependency, + CUDA OOM, version mismatch, or unexpected PTQ results. Trigger for "Quark error", "PTQ failed", + "quantization crashed", "CUDA out of memory", "import error", "model loading failed", "wrong + results", any Python traceback mentioning quark.torch / torch._dynamo / transformers / + accelerate. Not for onnxruntime tracebacks — use quark-onnx-debug. +--- + +# quark-torch-debug + +## Catalog portability + +This standalone skill is federated from `amd/Quark` at commit `1b229f781a1974cc742884e42d8eefc1eebb4f0a`. Resolve bundled files relative to this `SKILL.md`. Repository-relative paths such as `docs/`, `examples/`, and `quark/` refer to the [pinned Quark source](https://github.com/amd/Quark/tree/1b229f781a1974cc742884e42d8eefc1eebb4f0a); use a local Quark checkout when available, otherwise consult that pinned source. + +## Purpose + +Convert failures into a structured diagnostic report with the smallest safe recovery path. Debugging Quark issues is tricky because errors can originate from many layers — Python environment, PyTorch, transformers, CUDA/ROCm drivers, model architecture, or Quark itself. This skill systematically narrows down the root cause. + +## Inputs + +- Error message and stack trace from a failing run +- `env_context.json`, `pytorch_install_result.json`, `quark_install_result.json` (optional, for environment and install state) + +## Outputs: validation_report.md + +Diagnostic report with root cause, evidence, and the smallest safe fix. + +Schema: [`validation_report.schema.json`](https://github.com/amd/Quark/blob/1b229f781a1974cc742884e42d8eefc1eebb4f0a/.claude/skills-impl/shared/contracts/validation_report.schema.json) + +```markdown +# Debug Report + +## Common Error Patterns + +### Installation Errors + +| Error | Likely Cause | Fix | +|-------|-------------|-----| +| `ModuleNotFoundError: No module named 'quark'` | Quark not installed or wrong Python env | `pip install amd-quark` or activate correct conda env | +| `ImportError: quark.torch.kernel` | Missing C++ compiler | `sudo apt install build-essential` | +| `torch.cuda.is_available() == False` | CPU-only PyTorch installed | Reinstall PyTorch with correct `--index-url` | +| `RuntimeError: CUDA error: no kernel image` | PyTorch CUDA version ≠ system CUDA | Match PyTorch build to system CUDA version | + +### Model Loading Errors + +| Error | Likely Cause | Fix | +|-------|-------------|-----| +| `ValueError: Unrecognized model in config` | `trust_remote_code` needed | Add `--trust_remote_code` flag | +| `OSError: Can't load tokenizer` | Missing tokenizer files or sentencepiece | `pip install sentencepiece` and check model path | +| `ImportError: ... requires transformers>=X.Y.Z` | Transformers version too old | `pip install transformers==X.Y.Z` | +| `OutOfMemoryError` during loading | Model too large for single GPU | Use `--multi_gpu auto` or `--multi_device` | + +### Quantization Errors + +| Error | Likely Cause | Fix | +|-------|-------------|-----| +| `CUDA out of memory` during quantization | Insufficient GPU memory | Reduce `--num_calib_data` or `--batch_size`, or use `--multi_gpu` | +| `RuntimeError: expected scalar type Half` | Data type mismatch | Set `--data_type float16` or `bfloat16` explicitly | +| `KeyError: 'model.layers.0...'` | Model architecture not matching template | Check `model_type` in config.json matches Quark template | +| `ValueError: ... is not a valid quantization scheme` | Typo in scheme name | Check against the 21 supported schemes | +| `AssertionError` in AWQ/GPTQ | Algorithm config mismatch | Verify `--quant_algo_config_file` matches model architecture | + +### Export Errors + +| Error | Likely Cause | Fix | +|-------|-------------|-----| +| `ModuleNotFoundError: gguf` | GGUF package missing | `pip install gguf>=0.10.0` | +| `ONNX export failed` | Model has unsupported ops | Try `--model_export hf_format` instead | +| `PermissionError` on output dir | No write permission | Check output directory permissions | + +### Transformers Compatibility + +Quark checks compatibility before quantization. Known issues: +- `seen_tokens` removed in transformers > 4.53.3 +- `get_max_length` removed in transformers > 4.48.3 +- `get_usable_length` removed in transformers > 4.53.3 +- General constraint: `transformers < 5.3` + +## Diagnostic Process + +1. **Read the error** — get the exact error message, full stack trace, and the command that was run. +2. **Identify the layer** — is this an install problem, model loading, quantization runtime, or export issue? +3. **Check the environment** — Python version, PyTorch version, CUDA/ROCm version, Quark version, transformers version. +4. **Match against known patterns** — use the tables above. +5. **Propose the fix** — smallest change that resolves the issue without side effects. + +## Diagnostic Commands + +```bash +# Environment snapshot +python -c " +import sys; print('Python:', sys.version) +try: + import torch; print('PyTorch:', torch.__version__, 'CUDA:', torch.version.cuda, 'HIP:', torch.version.hip) +except: print('PyTorch: not installed') +try: + import quark; print('Quark:', quark.__version__) +except: print('Quark: not installed') +try: + import transformers; print('Transformers:', transformers.__version__) +except: print('Transformers: not installed') +" + +# GPU memory status +nvidia-smi --query-gpu=memory.used,memory.total --format=csv,noheader 2>/dev/null +rocm-smi --showmemuse 2>/dev/null + +# Check Quark compatibility +python -c " +from quark.torch.utils.llm.compatibility import check_compatibility_before_quantization +print('Compatibility check available') +" +``` + +## Rules + +- **Always ask for the full error message and the command that was run.** Partial errors lead to wrong diagnoses. +- **Do not guess the fix.** Narrow down the root cause first, then propose a specific solution. +- **Confirm before mutating.** If the fix involves reinstalling packages, changing environment, or modifying user files, present the plan and get confirmation. +- **Consider cascading effects.** Upgrading transformers might fix one issue but break compatibility with Quark. Always check version constraints. + +## Error Summary + +CUDA out of memory during quantization of Qwen/Qwen3-8B with FP8 + +## Root Cause + +Single GPU (24GB) insufficient for FP8 quantization with 512 calibration samples + +## Evidence + +- GPU memory: 22GB / 24GB used at crash point +- Model size: ~16GB in FP16 +- Calibration data: 512 samples at seq_len=512 + +## Fix + +Reduce calibration data or use multi-GPU: + +```bash +# Option A: Reduce calibration samples +python quantize_quark.py ... --num_calib_data 64 --batch_size 1 + +# Option B: Use multi-GPU (if available) +python quantize_quark.py ... --multi_gpu auto +``` + +## Prevention + +For models > 7B parameters on GPUs with < 48GB VRAM, start with +--num_calib_data 64 and increase if memory allows. + +```text + +## Interaction Flow + +1. **Gather evidence**: Get the error message, stack trace, command, and environment info. +2. **Classify**: Determine which layer failed (install / load / quantize / export). +3. **Diagnose**: Match against known patterns and run diagnostic commands if needed. +4. **Propose fix**: Present the smallest change that resolves the issue. +5. **Confirm**: Get user approval before any environment changes. +6. **Verify**: After the fix, re-run the failing step to confirm resolution. + +## Recovery + +- If the root cause is upstream (transformers API change, PyTorch bug), hand off to `the Quark repository's internal Torch documentation drift checker` or `the Quark repository's internal Torch skill synchronizer`. +- If the fix requires a PyTorch reinstall (wrong build, version mismatch, CPU instead of GPU), hand off to `quark-torch-install` with the specific requirement noted. +- If the fix requires a Quark package or dependency change, hand off to `quark-install` with the specific requirement noted. +- If the error is in a custom model or unsupported architecture, suggest registering a custom template via `LLMTemplate.register_template()`. diff --git a/skills/quark-torch-debug/skill-card.md b/skills/quark-torch-debug/skill-card.md new file mode 100644 index 0000000..ba9cca0 --- /dev/null +++ b/skills/quark-torch-debug/skill-card.md @@ -0,0 +1,13 @@ +# Skill Card + +## Description + +Diagnose failed Quark Torch PTQ installation, execution, script generation, or export attempts. Use when the user reports a torch-side error, stack trace, invalid artifact, missing dependency, CUDA OOM, version mismatch, or unexpected PTQ results. + +## Owner + +amd (federated from [amd/Quark](https://github.com/amd/Quark)) + +## License + +MIT diff --git a/skills/quark-torch-export/.federated.json b/skills/quark-torch-export/.federated.json new file mode 100644 index 0000000..783d7f1 --- /dev/null +++ b/skills/quark-torch-export/.federated.json @@ -0,0 +1,10 @@ +{ + "source": "amd-quark", + "repo": "amd/Quark", + "ref": "v0.12", + "commit": "1b229f781a1974cc742884e42d8eefc1eebb4f0a", + "path": ".claude/skills/quark-torch-export", + "license": "MIT", + "imported_at": "2026-08-05T21:07:39Z", + "implementation_path": ".claude/skills-impl/l1-atomic/torch/quark-torch-export" +} diff --git a/skills/quark-torch-export/SKILL.md b/skills/quark-torch-export/SKILL.md new file mode 100644 index 0000000..ea592fb --- /dev/null +++ b/skills/quark-torch-export/SKILL.md @@ -0,0 +1,167 @@ +--- +name: quark-torch-export +description: >- + Prepare export and downstream evaluation handoff for a planned or completed Quark Torch PTQ run. + Input is a PyTorch / HuggingFace transformers model; output formats include HF safetensors, + GGUF, and ONNX. Trigger for "export model", "save quantized model", "convert to GGUF", "export + to HuggingFace format", "export to ONNX", or when the user has a completed or planned Torch PTQ + run and needs deployment outputs. Not for exporting from an .onnx input model — use + quark-onnx-ptq. +--- + +# quark-torch-export + +## Catalog portability + +This standalone skill is federated from `amd/Quark` at commit `1b229f781a1974cc742884e42d8eefc1eebb4f0a`. Resolve bundled files relative to this `SKILL.md`. Repository-relative paths such as `docs/`, `examples/`, and `quark/` refer to the [pinned Quark source](https://github.com/amd/Quark/tree/1b229f781a1974cc742884e42d8eefc1eebb4f0a); use a local Quark checkout when available, otherwise consult that pinned source. + +When the workflow names one of these internal procedures, read its bundled reference before carrying out that step: + +- [`quark-torch-quant-plan`](references/quark-torch-quant-plan.md) + +## Purpose + +Translate a confirmed quantization plan into export expectations and downstream evaluation requirements. Export is a post-quantization step — the model must be quantized first (or have a plan to be quantized) before export decisions make sense. This skill ensures the right export format is chosen and evaluation is properly configured. + +## Inputs + +- `quant_plan.json` from quark-torch-quant-plan +- `workspace_context.json` for the output directory +- `run_manifest.yaml` (optional, existing manifest to extend) + +## Outputs: run_manifest.yaml + +Export does not own this artifact — it updates the workflow's manifest with export and evaluation config. + +Schema: [`run_manifest.schema.json`](https://github.com/amd/Quark/blob/1b229f781a1974cc742884e42d8eefc1eebb4f0a/.claude/skills-impl/shared/contracts/run_manifest.schema.json) + +(Export updates the workflow's `run_manifest.yaml` with export and evaluation fields rather than producing a new artifact.) + +```yaml +export: + formats: + - hf_format + output_dir: ./output/qwen3-8b-fp8 + weight_format: real_quantized + custom_mode: quark + +evaluation: + skip: false + metrics: + - ppl + dataset: wikitext + tasks: null + batch_size: auto +``` + +## Supported Export Formats + +### 1. HuggingFace SafeTensors (`hf_format`) — Default + +- Produces: `config.json` + `*.safetensors` files with `quantization_config` metadata +- Compatible with: HuggingFace transformers loading, vLLM, TGI +- CLI flag: `--model_export hf_format` +- Weight format options: + - `real_quantized` (default) — compressed, actual quantized weights + - `fake_quantized` — full-precision weights with quantization metadata only + +### 2. ONNX (`onnx`) + +- Produces: `quark_model.onnx` with optimization passes applied +- Compatible with: ONNX Runtime, TensorRT (with conversion) +- CLI flag: `--model_export onnx` +- Supports INT4/UINT4 conversion pass automatically + +### 3. GGUF (`gguf`) + +- Produces: GGUF format file for llama.cpp and compatible inference engines +- Requires: `gguf>=0.10.0` package and tokenizer path +- CLI flag: `--model_export gguf` +- Best with: `uint4_wo_32` scheme + AWQ algorithm + +Multiple formats can be exported simultaneously: `--model_export hf_format --model_export gguf` + +## Export CLI Arguments + +```bash +python quantize_quark.py \ + --model_dir /path/to/model \ + --output_dir /path/to/output \ + --quant_scheme fp8 \ + --model_export hf_format \ # Export format(s) + --export_weight_format real_quantized \ # Compression mode + --custom_mode quark \ # Export mode: quark|awq|fp8 + --pack_method reorder # Weight packing: order|reorder +``` + +## Evaluation Options + +Post-quantization evaluation can be configured as part of the export step: + +### Perplexity (PPL) + +- Default dataset: `wikitext` +- Flag: included by default unless `--skip_evaluation` is set + +### Task-Based Evaluation (via lm-eval harness) + +```bash +--tasks hellaswag,winogrande,arc_easy +--eval_batch_size auto +--num_fewshot 0 +``` + +### ROUGE/METEOR (for generation models) + +```bash +# Evaluated on cnn_dailymail by default +--use_mlperf_rouge # For MLPerf-compatible ROUGE scoring +``` + +### KV Cache Evaluation + +```bash +--use_ppl_eval_for_kv_cache +--ppl_eval_for_kv_cache_context_size 1024 +--ppl_eval_for_kv_cache_sample_size 512 +``` + +### Skip Evaluation + +```bash +--skip_evaluation # Skip all post-quantization evaluation +``` + +## Model Reload for Separate Evaluation + +If quantization and evaluation are done in separate steps: + +```bash +# Step 1: Quantize and export +python quantize_quark.py --model_dir MODEL --quant_scheme fp8 \ + --model_export hf_format --output_dir output/ --skip_evaluation + +# Step 2: Reload and evaluate +python quantize_quark.py --model_dir MODEL --model_reload \ + --output_dir output/ --skip_quantization +``` + +## Rules + +- **Never invent export paths** that conflict with the existing `run_manifest.yaml` or `quant_plan.json`. +- **Export depends on quantization.** If the model has not been quantized yet, this skill produces an export plan attached to the manifest — it does not run quantization. +- **Match export format to deployment target.** Ask the user where the model will run: HuggingFace ecosystem → `hf_format`, llama.cpp → `gguf`, ONNX Runtime → `onnx`. +- **GGUF works best with UINT4.** If the user wants GGUF but the plan uses FP8, flag the mismatch — GGUF is primarily designed for integer quantization. + +## Interaction Flow + +1. **Check prerequisites**: Is there a `quant_plan.json`? Has quantization been run or is this plan-only? +2. **Choose format**: Ask where the model will be deployed and recommend the right export format. +3. **Configure evaluation**: Ask if the user wants post-quantization evaluation and which metrics. +4. **Emit**: Update `run_manifest.yaml` with export and evaluation configuration. + +## Recovery + +- If export is requested before quantization, return the missing prerequisites and attach the export request to the manifest under `pending_exports`. +- If GGUF export fails, check that `gguf>=0.10.0` is installed and that the tokenizer is accessible. +- If ONNX export fails on a complex model, suggest trying `hf_format` first as a fallback. diff --git a/skills/quark-torch-export/references/quark-torch-quant-plan.md b/skills/quark-torch-export/references/quark-torch-quant-plan.md new file mode 100644 index 0000000..2694f3b --- /dev/null +++ b/skills/quark-torch-export/references/quark-torch-quant-plan.md @@ -0,0 +1,209 @@ +--- +name: quark-torch-quant-plan +description: > + Build a Quark Torch LLM PTQ quantization plan from model analysis and user intent. Use when the + user needs quantization scheme recommendations, exclusion lists, algorithm selection, KV cache + decisions, per-layer overrides, or a draft quant_plan. Trigger for "quantize with FP8", + "what scheme should I use", "plan PTQ", "INT4 quantization", "choose quantization config", + "quantization plan", or when the user has a model analysis and needs to decide how to quantize it. +layer: l1-atomic +primary_artifact: quant_plan.json +source_knowledge: + - quark/torch/quantization/config/template.py + - examples/torch/language_modeling/llm_ptq/README.md + - examples/torch/language_modeling/llm_ptq/quantize_quark.py +--- + +# quark-torch-quant-plan + +## Purpose + +Convert a model analysis plus the user's intent into a confirmed `quant_plan.json`. This skill makes the quantization decisions — which scheme, which algorithm, what to exclude — without generating scripts or executing PTQ. The plan is the contract between the user's intent and the execution step. + +## Inputs + +- `model_analysis.json` from quark-torch-model-intake +- `env_context.json` for accelerator-aware scheme recommendations +- User preferences (target precision, accuracy goal) + +## Outputs: quant_plan.json + +Records the chosen scheme, algorithm, layer overrides, calibration settings, and evaluation intent. + +Schema: [`quant_plan.schema.json`](https://github.com/amd/Quark/blob/1b229f781a1974cc742884e42d8eefc1eebb4f0a/.claude/skills-impl/shared/contracts/quant_plan.schema.json) + +```json +{ + "model": { + "model_type": "qwen3", + "analysis_ref": "./model_analysis.json" + }, + "global_scheme": "fp8", + "kv_cache_scheme": "fp8", + "exclude_layers": ["lm_head"], + "layer_quant_config": {}, + "algorithm": null, + "calibration": { + "dataset": "pileval", + "num_calib_data": 128, + "seq_len": 512 + }, + "evaluation_intent": "smoke", + "requires_confirmation": false +} +``` + +## Available Quantization Schemes (21 total) + +### Weight-Only INT4 (best for deployment size reduction) + +| Scheme | Description | Use Case | +|--------|-------------|----------| +| `int4_wo_32` | INT4, group size 32 | Highest accuracy among INT4 | +| `int4_wo_64` | INT4, group size 64 | Good balance | +| `int4_wo_128` | INT4, group size 128 | Smaller overhead | +| `int4_wo_per_channel` | INT4, per-channel | Least overhead | +| `uint4_wo_32/64/128/per_channel` | Unsigned INT4 variants | GGUF export compatibility | + +### Weight+Activation INT8 + +| Scheme | Description | Use Case | +|--------|-------------|----------| +| `int8` | INT8 per-tensor for both W and A | CPU deployment, good accuracy | + +### FP8 (best accuracy-size tradeoff for GPU inference) + +| Scheme | Description | Use Case | +|--------|-------------|----------| +| `fp8` | FP8 E4M3 per-tensor | Standard GPU quantization | +| `ptpc_fp8` | Per-Token-Per-Channel FP8 | Higher accuracy, dynamic activation quantization | + +### OCP Microscaling Formats + +| Scheme | Description | Use Case | +|--------|-------------|----------| +| `mxfp4` | OCP MXFP4 | Aggressive compression | +| `mxfp6_e3m2` | OCP MXFP6 (E3M2) | Better range | +| `mxfp6_e2m3` | OCP MXFP6 (E2M3) | Better precision | +| `mxfp4_mxfp6_e2m3` | MXFP4 weights + MXFP6 activations | Mixed precision | +| `mxfp4_fp8` | MXFP4 weights + FP8 activations | Mixed precision | + +### AMD-Specific + +| Scheme | Description | Use Case | +|--------|-------------|----------| +| `amdfp4` | amdfp4, group size 16 | AMD MI300X optimized | +| `amdfp4_g32` | amdfp4, group size 32 | AMD MI300X, less overhead | + +### Other + +| Scheme | Description | Use Case | +|--------|-------------|----------| +| `nvfp4` | NVFP4: FP4 group_size=16 with FP8 E4M3 scale | NVIDIA Blackwell/Hopper | +| `mx6` | MX6 format | Experimental | +| `bfp16` | Block Floating Point 16-bit | Experimental | +| `int4_wa_64` | INT4 weights + activations, group 64 | Research | + +## Available Algorithms (7 primary) + +| Algorithm | Compatible Schemes | Description | +|-----------|-------------------|-------------| +| `awq` | INT4/UINT4 weight-only | Activation-aware weight quantization — finds optimal per-channel scaling | +| `gptq` | INT4/UINT4 weight-only | Second-order weight optimization — often better than AWQ for small models | +| `smoothquant` | INT8, FP8 | Migrates quantization difficulty from activations to weights | +| `autosmoothquant` | INT8, FP8 | Automatic SmoothQuant with optimal alpha search | +| `rotation` | Various | Rotation-based optimization to equalize weight distribution | +| `gptaq` | INT4/UINT4 | GPTAQ variant combining GPTQ with activation quantization | +| `qronos` | Various | Custom algorithm for time-series-aware quantization | + +Algorithms can be combined: `--quant_algo awq,smoothquant` + +## KV Cache Quantization + +- Only `fp8` is supported for KV cache (`--kv_cache_dtype fp8`) +- Adds `--min_kv_scale` option (default 0.0) to prevent extreme scale values +- `--kv_cache_post_rope` quantizes KV cache after RoPE (inside cache) instead of at k_proj/v_proj outputs — can improve accuracy for some models + +## Decision Guide + +Help the user choose based on their priorities: + +**"I want the best accuracy"** → `fp8` or `ptpc_fp8`, optionally with `smoothquant` +**"I want the smallest model"** → `int4_wo_32` with `awq` or `gptq` +**"I need CPU deployment"** → `int8` (the only scheme that works well on CPU) +**"I need GGUF format"** → `uint4_wo_32` with `awq`, export as GGUF +**"I'm on AMD MI300X"** → `amdfp4` for best hardware utilization +**"I'm on NVIDIA H100/Blackwell"** → `fp8` or `nvfp4` +**"I want to experiment"** → `mxfp4` for aggressive compression research + +## Decision Table (MUST show to user) + +**ALWAYS present this table to the user and WAIT for confirmation before finalizing.** Do not skip this step. + +Fill in the "Value" column based on the user's request and model analysis, then show: + +| Decision | Value | Reason | +|----------|-------|--------| +| `global_scheme` | _(fill)_ | _(why this scheme)_ | +| `kv_cache_scheme` | _(fill: `fp8` or `null`)_ | _(explain)_ | +| `exclude_layers` | `["lm_head"]` | Standard — lm_head stays full precision | +| `layer_quant_config` | _(fill: dict of `pattern -> scheme`, or `{}` if none)_ | _(explain which patterns and why)_ | +| `algorithm` | _(fill: algorithm or `null`)_ | _(explain)_ | +| `calibration_dataset` | `pileval` | Fast default | +| `num_calib_data` | `128` | Standard default | +| `seq_len` | `512` | Standard default | +| `evaluation_intent` | `smoke` | Quick PPL check post-quantization | + +After showing the table, ask: "Confirm this plan? Any changes?" + +**Do NOT proceed until the user confirms.** + +## Layer-Specific Overrides via `layer_quant_config` + +The `layer_quant_config` plan field is a dict of `pattern -> scheme` pairs. It is the +single mechanism for any "quantize layer/module X with scheme Y" intent — including +attention modules, MoE experts, lm_head, etc. Each entry emits one +`--layer_quant_scheme PATTERN SCHEME` CLI argument. + +```json +"layer_quant_config": { + "*self_attn*": "fp8", + "lm_head": "int8", + "*experts*": "fp8" +} +``` + +translates to: + +```bash +--quant_scheme \ +--layer_quant_scheme '*self_attn*' fp8 \ +--layer_quant_scheme lm_head int8 \ +--layer_quant_scheme '*experts*' fp8 +``` + +When a user asks for attention-module quantization (e.g. "self_attn in fp8"), populate +this field with the appropriate pattern (commonly `*self_attn*` for LLaMA-style models; +adjust for models whose attention submodule has a different name). Do NOT introduce a +dedicated attention field — keep all per-pattern overrides in `layer_quant_config`. + +## Rules + +- **Keep scope to plan creation only.** Do not generate scripts, do not run quantization, do not export. Those are separate skills. +- **Require a model analysis first.** Without knowing the model architecture and layer count, you cannot make informed scheme recommendations. If `model_analysis.json` is missing, route back to `quark-torch-model-intake`. +- **Always present the decision table** before finalizing. The user should explicitly confirm the scheme, algorithm, and exclusions. +- **If a risky scheme is chosen** (e.g., `mxfp4` on a model where accuracy loss may be significant), keep the user's choice but record the risk in the plan. + +## Interaction Flow + +1. **Check prerequisites**: Is `model_analysis.json` available? If not, route to `quark-torch-model-intake` first. +2. **Gather intent**: What does the user care about most — accuracy, size, speed? What hardware will run inference? +3. **Present the decision table**: Show defaults, explain the tradeoffs, and let the user adjust. +4. **Confirm**: Always required. Show the final plan summary before writing it. +5. **Emit**: Write `quant_plan.json`. + +## Recovery + +- If the model analysis is incomplete, produce a draft plan with `requires_confirmation: true` and note what facts are missing. +- If the user picks an unusual combination (e.g., `awq` with `fp8` — AWQ is designed for INT4), explain why it might not work well and suggest alternatives, but respect the user's choice if they insist. +- If calibration dataset preferences are unclear, default to `pileval` with 128 samples — it is the fastest option and works for most models. diff --git a/skills/quark-torch-export/skill-card.md b/skills/quark-torch-export/skill-card.md new file mode 100644 index 0000000..1579447 --- /dev/null +++ b/skills/quark-torch-export/skill-card.md @@ -0,0 +1,13 @@ +# Skill Card + +## Description + +Prepare export and downstream evaluation handoff for a planned or completed Quark Torch PTQ run. Input is a PyTorch / HuggingFace transformers model; output formats include HF safetensors, GGUF, and ONNX. + +## Owner + +amd (federated from [amd/Quark](https://github.com/amd/Quark)) + +## License + +MIT diff --git a/skills/quark-torch-file2file-quantization/.federated.json b/skills/quark-torch-file2file-quantization/.federated.json new file mode 100644 index 0000000..a8635fb --- /dev/null +++ b/skills/quark-torch-file2file-quantization/.federated.json @@ -0,0 +1,10 @@ +{ + "source": "amd-quark", + "repo": "amd/Quark", + "ref": "v0.12", + "commit": "1b229f781a1974cc742884e42d8eefc1eebb4f0a", + "path": ".claude/skills/quark-torch-file2file-quantization", + "license": "MIT", + "imported_at": "2026-08-05T21:07:39Z", + "implementation_path": ".claude/skills-impl/l1-atomic/torch/quark-torch-file2file-quantization" +} diff --git a/skills/quark-torch-file2file-quantization/SKILL.md b/skills/quark-torch-file2file-quantization/SKILL.md new file mode 100644 index 0000000..2038030 --- /dev/null +++ b/skills/quark-torch-file2file-quantization/SKILL.md @@ -0,0 +1,209 @@ +--- +name: quark-torch-file2file-quantization +description: >- + Low-memory file2file quantization for very large safetensors LLMs that cannot be loaded whole. + Use when the user wants to run file2file quantization, adapt a new safetensors checkpoint + without loading the full model, register an external LLMTemplate, inspect sharded checkpoint + naming, generate wrapper or conversion scripts, or validate low-memory sharded quantization + outputs. Trigger for "run file2file quantization", "quantize without loading the model", "large + safetensors low-memory quantization", "file2file for DeepSeek/Qwen/MoE", "safetensors naming + incompatible", "register LLMTemplate externally". +--- + +# quark-file2file-quantization-runner + +## Catalog portability + +This standalone skill is federated from `amd/Quark` at commit `1b229f781a1974cc742884e42d8eefc1eebb4f0a`. Resolve bundled files relative to this `SKILL.md`. Repository-relative paths such as `docs/`, `examples/`, and `quark/` refer to the [pinned Quark source](https://github.com/amd/Quark/tree/1b229f781a1974cc742884e42d8eefc1eebb4f0a); use a local Quark checkout when available, otherwise consult that pinned source. + +## Purpose + +Quantize very large safetensors checkpoints without loading the full model into memory, using +`ModelQuantizer.direct_quantize_checkpoint()`. Covers the full adaptation path: checkpoint +inspection, external `LLMTemplate` registration, optional naming normalization via a one-time +conversion script, minimum-scale experiment gating, full file2file execution, and output +validation via `quark-torch-result-validator`. + +Default policy: solve naming or layer-selection mismatches with external adapters +(`LLMTemplate.register_template()`, `weight_converters`) or a temporary conversion script. +Do **not** modify Quark source unless the required capability is absent, the external path has +been ruled out, and the user explicitly agrees. + +## Inputs + +- `pretrained_model_path` — local directory of the safetensors checkpoint +- `save_path` — output directory +- `quant_scheme` — quantization scheme (e.g., `w_fp8_a_fp8`, `w_int4_a_bf16`) +- `device` — e.g., `cuda:0`, `cpu` +- User intent: direct quantization, checkpoint conversion first, HF-format export, or script generation only +- `model_analysis.json` from `quark-torch-model-intake` (optional but recommended) + +## Outputs: run_manifest.yaml + +Records the adaptation path taken, scripts generated, experiment results, and validation status. + +```yaml +pretrained_model_path: /models/DeepSeek-V3 +save_path: /output/DeepSeek-V3-fp8 +quant_scheme: w_fp8_a_fp8 +device: cuda:0 +adaptation_path: external_template # direct | external_template | weight_converters | conversion_script +conversion_script: null # path if generated +wrapper_script: /tmp/run_ds_v3_f2f.py +min_experiment: + status: passed # passed | failed | skipped + layers_covered: [layer_0_expert_0, ...] + moe_covered: true +full_run_status: completed # pending | completed | failed +validation_status: passed # passed | failed | not_run +``` + +## Interaction Flow + +### Step 1 — Inspect checkpoint + +Read keys and config without loading weights: + +```bash +python - <<'PY' +import json, os +from glob import glob +from safetensors.torch import safe_open + +model_dir = "" +index_path = os.path.join(model_dir, "model.safetensors.index.json") +print("config:", os.path.exists(os.path.join(model_dir, "config.json"))) +print("index:", os.path.exists(index_path)) +files = sorted(glob(os.path.join(model_dir, "*.safetensors"))) +print("safetensors:", len(files)) +if files: + with safe_open(files[0], framework="pt", device="cpu") as f: + keys = list(f.keys()) + print("sample_keys (first 80):") + for k in keys[:80]: print(" ", k) +if os.path.exists(index_path): + with open(index_path) as f: + wm = json.load(f).get("weight_map", {}) + print("index_keys:", len(wm)) +PY +``` + +Verify: `model_type`, weight-name suffixes (`*.weight`, `*_scale_inv`, `*.scale`), shard count, +MoE expert / shared-expert / gate naming, and whether scale tensors are co-located with weights. + +### Step 2 — Choose adaptation path + +| Situation | Action | +|-----------|--------| +| Names already match Quark template | Direct file2file; tune `exclude_layers` only | +| Layer naming differs from built-in template | External `LLMTemplate.register_template()` | +| Only weight suffixes differ | `weight_converters` / `_apply_weight_converters` | +| Scale naming or dtype incompatible pre-recovery | Generate normalization conversion script first | + +`_apply_weight_converters` limits: suited for post-recovery single-suffix rename or one-source split. +Not suited for multi-source merge, cross-shard scale pairing, or FP4→FP8 dtype conversion. + +### Step 3 — Generate wrapper or conversion script + +For external template registration, generate a wrapper script (do NOT modify `quantize_quark.py`): + +```python +from quark.torch import ModelQuantizer +from quark.torch.utils.llm import LLMTemplate + +template = LLMTemplate( + model_type="", + kv_layers_name=[""], + q_layer_name=[""], + exclude_layers_name=["embed", "head", ""], +) +LLMTemplate.register_template(template) + +quantizer = ModelQuantizer(config) +quantizer.direct_quantize_checkpoint( + pretrained_model_path=pretrained_model_path, + save_path=save_path, + keep_excluded_layers_as_original_model_state=False, + weight_converters=weight_converters, + device=device, +) +``` + +Wrapper must print: registered `model_type`, input/output dirs, quant scheme, and exclude rules. +Conversion scripts must stream safetensors (no full-model load), include explicit `remap_name()`, +scale/weight pairing validation, shard output in HF style, index rebuild, and atomic output. + +### Step 4 — Minimum-scale experiment (mandatory gate) + +This step is not optional. Full file2file must not run until the minimum experiment passes. + +Construct the minimum input: + +1. If `num_hidden_layers` is safely reducible, copy `config.json` with the smallest value that + still covers at least one MoE layer (for MoE models, use `first_moe_layer_id + 1`). +2. If not, generate a subset checkpoint filtered by `--key-regex` covering at least one complete + MoE expert + its scale tensor + shared expert/router/gate + adjacent non-quantized tensors. +3. Never set `num_hidden_layers=1` for a MoE model if layer 0 is dense — verify from config or + key patterns which layer is the first actual MoE layer. + +Run the minimum experiment, then call `quark-torch-result-validator` with: +`inspect_safetensors`, `summarize_dtypes`, `check_index_consistency`, `check_scale_pairs`, +`get_fuzzy_tensor_names`, and auxiliary-file copy check (if source dir available). + +If validation fails, fix template / naming / subset and re-run. Do not proceed to full file2file. + +### Step 5 — Full file2file + +Set cache paths to avoid polluting home quota: + +```bash +export TMPDIR=/path/to/run/tmp +export TORCH_EXTENSIONS_DIR=/path/to/run/torch_extensions +export TRITON_CACHE_DIR=/path/to/run/triton_cache +``` + +Run the wrapper script. After completion, re-run `quark-torch-result-validator` on the +final output with the same checks as Step 4. + +### Step 6 — Deliver + +Emit `run_manifest.yaml` and present to the user: + +- Script paths and execution commands +- Minimum experiment summary: layers selected, MoE coverage, validation outcome +- Full run validation summary +- Any outstanding risk items + +## Recovery + +| Failure | Action | +|---------|--------| +| Missing Triton / compressed-tensors | Install dependency, retry | +| Incomplete source shards | Repair shards and index before proceeding | +| Template not recognized | Register via external `LLMTemplate`; do not patch Quark source | +| Scale/weight cannot be paired | Normalize checkpoint first; add unit test if Quark recovery is extended | +| Output index inconsistent | Rebuild index or fix shard write logic; re-validate | +| Minimum experiment fails | Fix adapter/naming/subset; never skip to full run | + +## Rules + +- **Never load full weights** for inspection — read safetensors headers only. +- **External adapters first**: use `LLMTemplate.register_template()` or `weight_converters` before + any Quark source change. +- **Minimum experiment is a gate, not a hint** — full file2file is blocked until it passes. +- **MoE minimum experiment** must include actual MoE expert weights + their scale tensors. + Setting `num_hidden_layers=1` when layer 0 is dense is invalid. +- **No cluster paths in shared scripts** — keep one-time paths in temporary wrapper scripts only. +- **Source changes require tests** — if Quark source must change, add a `test/test_for_torch/` + unit test and run the relevant pytest before committing. + +## Notes + +- DeepSeek-V4-family: prefer external `LLMTemplate`; check whether inference-format naming needs + a conversion script (`embed_tokens→embed`, `self_attn→attn`, `q_proj→wq`, etc.) before file2file. +- FP4/e2m1fn expert weights must be converted to FP8/e4m3fn in the conversion script, not via + `_apply_weight_converters`. +- Sibling scale naming (`{base}.scale`) must be resolvable before recovery; post-recovery suffix + conversion cannot substitute for pre-recovery scale identification. +- MTP embedding/head, attention, gate, `hc_*` auxiliary tensors must be explicitly excluded in + the template or conversion script. diff --git a/skills/quark-torch-file2file-quantization/skill-card.md b/skills/quark-torch-file2file-quantization/skill-card.md new file mode 100644 index 0000000..79d6a49 --- /dev/null +++ b/skills/quark-torch-file2file-quantization/skill-card.md @@ -0,0 +1,13 @@ +# Skill Card + +## Description + +Low-memory file2file quantization for very large safetensors LLMs that cannot be loaded whole. Use when the user wants to run file2file quantization, adapt a new safetensors checkpoint without loading the full model, register an external LLMTemplate, inspect sharded checkpoint naming, generate wrapper or conversion sc… + +## Owner + +amd (federated from [amd/Quark](https://github.com/amd/Quark)) + +## License + +MIT diff --git a/skills/quark-torch-install/.federated.json b/skills/quark-torch-install/.federated.json new file mode 100644 index 0000000..b156f2a --- /dev/null +++ b/skills/quark-torch-install/.federated.json @@ -0,0 +1,10 @@ +{ + "source": "amd-quark", + "repo": "amd/Quark", + "ref": "v0.12", + "commit": "1b229f781a1974cc742884e42d8eefc1eebb4f0a", + "path": ".claude/skills/quark-torch-install", + "license": "MIT", + "imported_at": "2026-08-05T21:07:39Z", + "implementation_path": ".claude/skills-impl/l1-atomic/torch/quark-torch-install" +} diff --git a/skills/quark-torch-install/SKILL.md b/skills/quark-torch-install/SKILL.md new file mode 100644 index 0000000..bab6681 --- /dev/null +++ b/skills/quark-torch-install/SKILL.md @@ -0,0 +1,121 @@ +--- +name: quark-torch-install +description: >- + Install or verify the correct PyTorch build for a user's accelerator backend before Quark + installation. Trigger for "install PyTorch", "pip install torch", "set up torch for ROCm", "set + up torch for CUDA", "torch version mismatch", "CPU-only torch installed", + torch.cuda.is_available() returns False, or any request to get the correct PyTorch build + running. +--- + +# quark-torch-install + +## Catalog portability + +This standalone skill is federated from `amd/Quark` at commit `1b229f781a1974cc742884e42d8eefc1eebb4f0a`. Resolve bundled files relative to this `SKILL.md`. Repository-relative paths such as `docs/`, `examples/`, and `quark/` refer to the [pinned Quark source](https://github.com/amd/Quark/tree/1b229f781a1974cc742884e42d8eefc1eebb4f0a); use a local Quark checkout when available, otherwise consult that pinned source. + +## Purpose + +Install the correct PyTorch build for the user's accelerator backend. PyTorch must be installed before Quark because Quark depends on PyTorch at both install time and runtime. Getting this wrong — installing the CPU build on a GPU machine, or mixing a CUDA-built PyTorch with a ROCm environment — causes cryptic failures that are hard to diagnose later. This skill exists separately from `quark-install` so that PyTorch setup has a clear, single-responsibility boundary. + +## Inputs + +- `env_context.json` with detected accelerator info + +## Outputs: pytorch_install_result.json + +Records the installed PyTorch build, accelerator backend tag, and verification status. + +Schema: [`pytorch_install_result.schema.json`](https://github.com/amd/Quark/blob/1b229f781a1974cc742884e42d8eefc1eebb4f0a/.claude/skills-impl/shared/contracts/pytorch_install_result.schema.json) + +```json +{ + "status": "ok", + "pytorch_version": "2.5.1+cu126", + "accelerator_tag": "cu126", + "torchvision_version": "0.20.1+cu126", + "torchaudio_version": "2.5.1+cu126", + "verification": { + "import_ok": true, + "cuda_available": true, + "gpu_count": 1 + } +} +``` + +On failure, set `status: "failed"` and include a `failure_reason` with the exact failing verification command. + +## Python Version Requirements + +- **Supported**: Python 3.11, 3.12, 3.13 +- **Not supported**: Python 3.14+ +- **Recommended for new setups**: Python 3.13 via Miniforge/Miniconda + +## PyTorch Version Matrix + +**Authoritative source**: `tools/ci/install_torch.sh` + +Before generating install commands, always read this script to get the current list of verified accelerator/PyTorch combinations. The script defines which PyTorch versions are tested with each accelerator backend (ROCm, CUDA, CPU) and the corresponding `--index-url` values. + +### How to read the source + +1. Open `tools/ci/install_torch.sh` and locate the version arrays or case/if blocks that map accelerator tags to PyTorch versions. +2. Extract the accelerator identifier (e.g., `rocm7.1`, `cu126`, `cpu`). +3. Extract the supported PyTorch version list for that accelerator. +4. Construct the install command using the pattern below. + +### Install command pattern + +```bash +# ROCm — torchvision only, no torchaudio +pip install torch torchvision --index-url https://download.pytorch.org/whl/ + +# CUDA — includes torchaudio +pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/ + +# CPU — includes torchaudio +pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu +``` + +Replace `` with the ROCm version tag (e.g., `rocm6.4`, `rocm7.0`, `rocm7.1`) and `` with the CUDA version tag (e.g., `cu118`, `cu126`, `cu128`, `cu130`). Always use the exact tags from `install_torch.sh`. + +**Critical**: Never use bare `pip install torch` for GPU setups — it installs the CPU version by default. + +## Rules + +- **Always read `tools/ci/install_torch.sh` before generating install commands.** The version matrix changes with each Quark release. Never rely on memorized version numbers — always verify against the upstream script. +- **Always detect the accelerator before choosing the PyTorch package.** Run or reference `quark-env-preflight` if hardware facts are missing. The entire install plan depends on getting this right. +- **Bind torch and accelerator to the same backend.** Never mix a CUDA-built PyTorch with ROCm environment or vice versa. If `torch.version.cuda` shows a CUDA version but the user says they want ROCm, flag the conflict. +- **Never skip verification.** After installation, always run verification commands. +- **If accelerator is unclear, stop after the plan.** Present the install plan but do not execute. Hand the gap back to `the matching public quark-torch skill` so it lands in `session_context.json`'s `open_questions`, and ask the user to confirm their hardware. +- **Show exact commands before execution.** The user should see every `pip install` command, every version, and every `--index-url` before anything runs. + +## Verification Commands + +```bash +# PyTorch backend check +python -c "import torch; print('PyTorch:', torch.__version__); print('CUDA:', torch.version.cuda); print('HIP:', torch.version.hip)" + +# GPU availability +python -c "import torch; print('CUDA available:', torch.cuda.is_available()); print('GPU count:', torch.cuda.device_count())" +``` + +## Interaction Flow + +1. **Intake**: Determine what the user already has installed and what accelerator they need. Check if `quark-env-preflight` has already run. +2. **Plan**: Present the accelerator-specific PyTorch installation command with version justifications. +3. **Confirm**: Required before any package installation. Show: what will be installed, which `--index-url` will be used, and what environment will be modified. +4. **Execute**: Run the installation commands. +5. **Verify**: Run all verification commands. Report pass/fail for each. + +## Recovery + +- **If `torch.cuda.is_available() == False`**: PyTorch CPU build was installed instead of GPU build. Show the exact uninstall + reinstall commands with the correct `--index-url`. +- **If torch and accelerator mismatch**: Explain that PyTorch must be reinstalled with the correct `--index-url`. Show the exact uninstall + reinstall commands. +- **If Python version is wrong**: Recommend creating a new conda environment with a supported version (3.11, 3.12, or 3.13). + +## Windows-Specific Notes + +- If pip fails with long path errors: Enable Win32 long paths via Group Policy Editor (Computer Configuration > Administrative Templates > System > Filesystem > Enable Win32 long paths) +- WSL2 with Ubuntu is recommended as an alternative for Windows users +- ROCm is not supported on Windows — only CUDA and CPU diff --git a/skills/quark-torch-install/skill-card.md b/skills/quark-torch-install/skill-card.md new file mode 100644 index 0000000..3a4f078 --- /dev/null +++ b/skills/quark-torch-install/skill-card.md @@ -0,0 +1,13 @@ +# Skill Card + +## Description + +Install or verify the correct PyTorch build for a user's accelerator backend before Quark installation. Trigger for "install PyTorch", "pip install torch", "set up torch for ROCm", "set up torch for CUDA", "torch version mismatch", "CPU-only torch installed", torch.cuda.is_available() returns False, or any request to … + +## Owner + +amd (federated from [amd/Quark](https://github.com/amd/Quark)) + +## License + +MIT diff --git a/skills/quark-torch-llm-eval/.federated.json b/skills/quark-torch-llm-eval/.federated.json new file mode 100644 index 0000000..0f18343 --- /dev/null +++ b/skills/quark-torch-llm-eval/.federated.json @@ -0,0 +1,10 @@ +{ + "source": "amd-quark", + "repo": "amd/Quark", + "ref": "v0.12", + "commit": "1b229f781a1974cc742884e42d8eefc1eebb4f0a", + "path": ".claude/skills/quark-torch-llm-eval", + "license": "MIT", + "imported_at": "2026-08-05T21:07:39Z", + "implementation_path": ".claude/skills-impl/l1-atomic/torch/quark-torch-llm-eval" +} diff --git a/skills/quark-torch-llm-eval/SKILL.md b/skills/quark-torch-llm-eval/SKILL.md new file mode 100644 index 0000000..177a060 --- /dev/null +++ b/skills/quark-torch-llm-eval/SKILL.md @@ -0,0 +1,351 @@ +--- +name: quark-torch-llm-eval +description: >- + End-to-end LLM accuracy evaluation on AMD ROCm (ROCm-only) — docker container setup OR host + (no-docker) runtime, vLLM/SGLang/ATOM serving, lm-eval / lighteval / evalscope benchmarks. Use + when the user wants to evaluate, benchmark, or compare an LLM's accuracy. Trigger for "evaluate + this model", "run gsm8k/mmlu/mmlu_pro/aime/gpqa/hellaswag/arc", "test accuracy", "measure + perplexity", "compare quantized model accuracy", "does this mxfp4 model lose accuracy". For + evaluating Quark Agent Skills themselves, use the Quark repository's internal skill evaluator + instead. +--- + +# LLM Evaluation on AMD ROCm + +## Catalog portability + +This standalone skill is federated from `amd/Quark` at commit `1b229f781a1974cc742884e42d8eefc1eebb4f0a`. Resolve bundled files relative to this `SKILL.md`. Repository-relative paths such as `docs/`, `examples/`, and `quark/` refer to the [pinned Quark source](https://github.com/amd/Quark/tree/1b229f781a1974cc742884e42d8eefc1eebb4f0a); use a local Quark checkout when available, otherwise consult that pinned source. + +End-to-end LLM accuracy evaluation on AMD ROCm — from Docker container to final accuracy table. + +## When to use + +Trigger this skill when the user asks to: + +- Evaluate, benchmark, or test a model's accuracy +- Run gsm8k / mmlu / mmlu_pro / aime / math_500 / gpqa / hellaswag / arc +- Deploy a model with vLLM, SGLang, or ATOM and measure accuracy +- Compare quantized model quality against a reference +- Verify a newly quantized model ("does this mxfp4 model lose accuracy?") + +## Inputs + +| Parameter | Required | Description | +|-----------|----------|-------------| +| `model_path` or `hf_id` | **Yes** | Local path to model weights, or HuggingFace model ID | +| `benchmark` | No | Benchmark to run (default: `gsm8k`) | +| `backend` | No | Serving backend: `vLLM` (default), `SGLang`, or `ATOM` | +| `image` | No | Docker image (default: `vllm/vllm-openai-rocm:latest`; overridden by model card / Recipes / user). Ignored when `runtime=host`. | +| `runtime` | No | `docker` (default) or `host`. Auto-detected in Phase 0.5: `docker` when the daemon is reachable, else `host` when a native `vllm` / `sglang` is importable. ATOM is docker-only. Stop and report if neither path is available. | + +## Outputs + +`$EVAL_STATE_DIR/eval_report.md` — see `templates.md` §2 for the full skeleton, formatting rules, and the example used as the canonical shape. + +## Design Principles + +1. **Trust the model card** — launch commands from model card / Recipes / Cookbook are author-verified. Use them as-is, only adjusting TP and `HIP_VISIBLE_DEVICES`. Do not preemptively stack flags. +2. **Reactive error handling** — `troubleshooting.md` is a diagnostic guide, not a prevention checklist. Launch first, consult on failure. + +## Interaction Flow + +### Unified confirmation gate + +After Phase 0 + 0.5 + 1 finish collecting all information, present a **single confirmation summary** to the user before any container or server operations begin. Use the template in `templates.md` §1 — it defines all required fields, command format, and display rules. + +Wait for explicit user approval before executing any of these commands. This avoids piecemeal confirmations and lets the user review the full pipeline at once. + +**After approval**, the following are pre-approved by the unified gate — execute without re-asking: + +- `docker run` / `docker exec` / `docker pull` of the planned container/image +- The exact `vllm serve` / `python -m sglang.launch_server` / `python -m atom.entrypoints.openai_server` command shown in the plan (with the planned GPU/TP/port) +- The exact `lm_eval` / `lighteval` / `evalscope` command shown in the plan +- `pip install` of frameworks listed in `eval-frameworks.md` §Installation — **docker runtime only** (inside the planned container). In host runtime, pip install is never pre-approved; see Phase 0.5 host-mode constraints. + +Each of the following **still requires per-command confirmation**, even after the unified gate: + +- Any edit to model `config.json` or backend source code (show full diff first) +- `kill -9` / `sudo` / `rm -rf` / `mv` outside `$EVAL_STATE_DIR` and outputs +- Any `vllm serve` / eval command that **deviates** from the plan (different flags, different port, retry with new args) +- `docker stop` / `docker rm` (Phase 7 cleanup — always ask) + +Read-only probes (`ls`, `cat`, `du`, `rocm-smi`, `docker ps`, `docker images`, `curl -sI`, `curl -fsSL`, Read) do **not** need confirmation. + +### Never kill other users' processes + +- Never use `pkill -f ` or `ps | grep | awk | kill` broad matching +- Verify any PID is alive and belongs to the expected backend via `kill -0 ` + `ps -fp ` (check `USER` and `CMD` columns) before sending a signal — PIDs are recycled +- Send `SIGTERM` first, wait up to 30 s, then `SIGKILL` only if still alive + +### HuggingFace fetch rules + +All HuggingFace HTTP requests follow `model-inspection.md` §2 — `curl -fsSL`, `${HF_ENDPOINT:-https://huggingface.co}`, token from `~/.hf_token` then `$HF_TOKEN`. Read that section before the first HF request in any phase. + +### Phase 0 — Auto-discover + +**Prerequisite: user must provide `model_path` (local path) or `hf_id` (HuggingFace ID).** + +- **local path** → verify it exists (`ls /config.json`) → proceed +- **hf_id only** → fetch `config.json` and `tokenizer_config.json` from HF API (use the HuggingFace fetch rules above), then continue to Phase 0.5 → 1. Before Phase 2, ask: **A** "I have a local copy" or **B** "Download from HuggingFace" +- **model name only** (e.g., "llama 7b") → search HuggingFace, list candidates, let user choose → handle as hf_id +- **nothing** → ask for model_path or hf_id + +Once a local path is confirmed, read `model-inspection.md` and follow its rules (infer hf_id, parse config.json, detect model type, estimate size, compute TP candidates). Write findings to `$EVAL_STATE_DIR/current-eval.yaml`. + +### Phase 0.5 — Environment preflight (fail-fast) + +Before Phase 1, run a 60-second environment probe to fail fast: + +```bash +which curl python3 +# Probe both runtimes; do NOT fail if docker is missing — host mode covers that. +docker --version >/dev/null 2>&1 && docker info >/dev/null 2>&1 \ + && echo "docker_available=true" || echo "docker_available=false" +python3 -c "import vllm; print('vllm', vllm.__version__)" 2>/dev/null \ + && echo "host_vllm_available=true" || echo "host_vllm_available=false" +python3 -c "import sglang; print('sglang', sglang.__version__)" 2>/dev/null \ + && echo "host_sglang_available=true" || echo "host_sglang_available=false" +``` + +**Runtime selection (record `runtime:` in `current-eval.yaml`):** + +1. If the caller passed `runtime` explicitly (`docker` or `host`), honour it; validate the matching tool exists. +2. Else if `docker_available=true`, use `docker` (default — production path). +3. Else if `backend=vLLM` and `host_vllm_available=true`, or `backend=SGLang` and `host_sglang_available=true`, fall back to `runtime=host` and **report the fallback in the unified confirmation**. +4. Else stop and report: docker is not available, and the requested backend is not importable on the host either. List which of (docker, vllm, sglang) is missing and instruct the user to install one. **ATOM is docker-only** — if `backend=ATOM` and docker is missing, stop here regardless of host imports. + +Host-mode constraints (announce in the unified confirmation): + +- `image` is ignored; the host's currently-installed framework version is what runs. +- All later phases that show `docker exec ${CONTAINER} ` become `` directly on the host (the "Host runtime variant" call-out under each phase below). +- `pip install` of frameworks is NOT performed — the host environment is used as-is. Missing a benchmark framework (e.g., `lm_eval`) → stop and ask the user to install it explicitly; do not pip install on the host without consent. + +**Session path setup (first thing in this phase):** generate a unique, per-session state directory to avoid collisions between concurrent evals. + +`bash` shell state does NOT persist across separate `Bash` tool calls — `EVAL_USER` / `EVAL_TS` exported in one call are gone in the next. The path must therefore be **resolved once and persisted to disk**, not re-exported each phase. + +```bash +EVAL_USER=$(whoami) +EVAL_TS=$(date +%Y%m%d_%H%M%S) +EVAL_STATE_DIR="/tmp/eval-state/${EVAL_USER}/${EVAL_TS}" +mkdir -p "${EVAL_STATE_DIR}" +echo "eval_state_dir: ${EVAL_STATE_DIR}" > "${EVAL_STATE_DIR}/current-eval.yaml" +echo "${EVAL_STATE_DIR}" > /tmp/eval-state/.last-session-path-${EVAL_USER} +``` + +All subsequent state files live under `$EVAL_STATE_DIR`: + +- `$EVAL_STATE_DIR/current-eval.yaml` — phase progress (machine-readable YAML) +- `$EVAL_STATE_DIR/server-runtime.md` — server state +- `$EVAL_STATE_DIR/vllm-server.log` — server log (redirected here) +- `$EVAL_STATE_DIR/eval-output.log` — eval framework output +- `$EVAL_STATE_DIR/monitor.pid` — background monitor task IDs (Phase 3) + +**Recovering `$EVAL_STATE_DIR` in later phases**: every later Bash call starts with + +```bash +EVAL_STATE_DIR=$(cat /tmp/eval-state/.last-session-path-$(whoami)) +``` + +or reads the value from the first line of the most recent `current-eval.yaml`. Do not call `date +%Y%m%d_%H%M%S` again — it produces a different timestamp and a different (empty) directory. + +Record results in the `env_preflight` section of the state file (capture all probe outputs plus the chosen `runtime`). If neither `docker run` nor a native backend is usable per the Runtime selection rules above → **stop and report**, listing what is missing. + +### Phase 1 — Info collection (fully autonomous) + +**Do not ask the user any questions during Phase 0, 0.5, or 1.** Collect all information autonomously, apply sensible defaults for anything missing, and present everything in a single unified confirmation at the end. The user makes one decision — approve or adjust. + +Defaults for missing info: + +- `image`: no model card / recipe specifies one → default by backend: vLLM `vllm/vllm-openai-rocm:latest`; ATOM `rocm/atom-dev:latest`; SGLang `lmsysorg/sglang:v0.5.11-rocm720-mi35x` +- `benchmark`: unspecified and no model card recommendation → default to `gsm8k` +- `backend`: unspecified → default to `vLLM` + +**Image availability check**: run `docker images --format '{{.Repository}}:{{.Tag}}'` and check if the chosen image already exists locally. Report the result in the unified confirmation (e.g., "Image: vllm/vllm-openai-rocm:latest (already pulled)" or "Image: vllm/vllm-openai-rocm:latest (needs pull)"). + +Fetch model card and look for: recommended Docker image, serving / launch command, evaluation command, recommended benchmark. + +**Model card fetch**: always use `curl` to get the raw README.md directly — returns raw markdown without needing HTML parsing or JS rendering. Apply the HuggingFace fetch rules above: + +```bash +curl -fsSL "${HF_BASE}//raw/main/README.md" \ + ${HF_TOKEN:+-H "Authorization: Bearer $HF_TOKEN"} +``` + +**Fetch strategy — per-item priority chain with two-stage model card lookup:** + +Five pieces of info are collected: `image`, `launch_cmd`, `eval_cmd`, `eval_settings`, and `reference_scores`. Each one independently walks the priority chain below. Finding one item at a given level does NOT skip that level for other items — only skip lower levels for the specific item already found. + +`eval_settings` includes: `num_fewshot`, evaluation mode (`base`/`chat`, i.e. `local-completions`/`local-chat-completions`), and prompting strategy (`CoT`/`direct`). These settings determine how the eval command is constructed and are critical for reproducing published results. + +Example: model card provides `launch_cmd` but not `eval_cmd` → `launch_cmd` is settled. For `eval_cmd`, continue down the chain until found. + +Priority levels: + +1. **Quantized model's own model card** — `curl -sL https://huggingface.co//raw/main/README.md` (e.g., `amd/MiniMax-M2.7-mxfp4`). Highest trust: quant-specific commands verified by the quantized model's author. + +2. **Sibling quant variant's model card** — if the quantized model has no HF page or no useful commands, search for sibling quant variants under the same org: + + ```bash + curl -fsSL "${HF_BASE}/api/models?search=/&author=" \ + ${HF_TOKEN:+-H "Authorization: Bearer $HF_TOKEN"} + ``` + + If a variant with a different quant strategy is found (e.g., `amd/Model-fp8` when evaluating `amd/Model-mxfp4`), fetch its model card. Commands from a sibling variant share the same architecture and only differ in quantization flags. Record `launch_cmd_source: similar_quant_card` and `reference_model: `. + +3. **Base model's model card** — `curl -sL https://huggingface.co//raw/main/README.md` (e.g., `MiniMaxAI/MiniMax-M2.7`). Commands need adaptation: replace model path, may need quantization-specific adjustments. Record `launch_cmd_source: base_model_card`. + +4. **vLLM Recipes** — use `base_hf_id` (base model org/name, not quantized variant). Recipes pages are JS-rendered; try `curl -sL https://recipes.vllm.ai//` and extract from `__NEXT_DATA__` or HTML. If not accessible, skip to next level. +5. **ATOM Recipes** / **SGLang Cookbook** — backend-specific sources +6. **Framework repos** / **InferenceX** — eval commands only +7. **Paper / technical report** — for `eval_cmd` and `eval_settings` only. When levels 1–6 do not provide an explicit eval command or eval settings, extract settings from the model's arxiv paper. See `eval-frameworks.md` §Paper Eval Settings Extraction for details. Record `eval_cmd_source: paper` and `eval_settings_source: paper`. +8. **Generic templates** from `eval-frameworks.md` — last resort. Record `launch_cmd_source: template`. + +Record each item and its source in the state file (e.g., `launch_cmd_source: quant_model_card`, `eval_cmd_source: template`). + +**Reference scores collection**: while fetching model cards and papers in the priority chain above, also collect benchmark reference scores for the target benchmark(s). Walk the lookup chain in `eval-frameworks.md` §Accuracy Comparison — it covers model card, base model card, paper/technical report, Open LLM Leaderboard, and similar-scale fallback. Record the score, source, and **eval setting** (num_fewshot, CoT/direct, base/chat) for each benchmark. This enables the unified confirmation to show expected accuracy and flag setting mismatches before the eval runs. + +**Note:** The local path may be a quantized variant (e.g., `/shareddata/amd/Model-mxfp4`), but Recipes uses the base model ID (`base_hf_id`). In Phase 3, replace the base model ID in the command with the local path. + +### Phase 2 — Container + +**Host runtime variant:** if `runtime=host` (set in Phase 0.5), **skip this phase entirely**. Set `CONTAINER=""` (sentinel) in the state file so Phase 3+ branches detect host mode. Jump straight to Phase 3. + +**Docker runtime (default):** Read `backends.md` for the full `docker run` template. Name the container `eval---` (username from `whoami`). + +### Phase 3 — Pick & adapt launch command + +**Starting point — use the highest-priority source found in Phase 1:** + +1. Model card command → use as-is (highest trust) +2. Recipes / Cookbook command → use if model card had none +3. No command from any source → minimal template by backend: + - **vLLM**: `vllm serve --tensor-parallel-size ` + - **ATOM**: `python -m atom.entrypoints.openai_server --model --kv_cache_dtype fp8 -tp ` + - **SGLang**: `python -m sglang.launch_server --model-path --tp ` + +**Model card command rule**: when a model card provides a launch command, preserve ALL flags from that command. The allowed/forbidden modifications table below only applies to changes YOU make on top of the model card command — do not strip flags that the model card author included (e.g., `--enforce-eager`, `--reasoning-parser`, `--mm-encoder-tp-mode`, env vars like `VLLM_ROCM_USE_AITER=1`). + +**Adaptation rules**: see `backends.md` §Cross-Backend Launch Adaptation Rules — applies to all three backends. Only the changes in that table are allowed on top of the source command; reactive flags (`--enforce-eager` etc.) come from `troubleshooting.md` only when an error matches. + +**TP priority**: see `model-inspection.md` §5 — recipe TP takes precedence over auto-inferred. + +**ATOM-specific notes:** ATOM uses `--kv_cache_dtype fp8` by default for memory efficiency. ATOM auto-detects quantization configs from HuggingFace model configs (FP8, MXFP4, INT8, INT4). For MTP speculative decoding, add `--method mtp --num-speculative-tokens 3`. ATOM also supports running as a vLLM plugin — if `atom` is installed alongside `vllm`, `vllm serve` automatically picks up ATOM's optimized kernels. + +**Pre-launch checks (immediately before launching):** + +1. GPU index mapping: follow `model-inspection.md` §4 — build the rocm-smi → HIP index mapping via PCIe bus addresses. `rocm-smi` indices and `HIP_VISIBLE_DEVICES` indices are NOT the same. +2. GPU availability: use `rocm-smi --showmeminfo vram` for exact free VRAM, `--showuse` for GPU%, `--showpids` for processes. Identify idle GPUs by rocm-smi index, then convert to HIP indices using the mapping. +3. Select final TP: smallest value from Phase 0 TP candidates that fits the available idle GPU count and whose aggregate free VRAM ≥ model size × 1.3 +4. Port selection: default to **8080** (not 8000 — commonly occupied by other services). If model card specifies a port, use that instead. Check with `lsof -i :` — if occupied, try 8081, 8082, etc. Add `--port ` to the launch command. +5. Write chosen GPUs (both rocm-smi and HIP indices), TP, and port to state file + +**Launch template — server log path is mandatory.** Phase 4/5/7 monitoring all assume the log is reachable at `${EVAL_STATE_DIR}/vllm-server.log`; do not deviate. + +Both runtimes write the server log directly to `${EVAL_STATE_DIR}/vllm-server.log` — for docker, this works because the `docker run` template in `backends.md` bind-mounts `${EVAL_STATE_DIR}` into the container at the same path. + +**Docker runtime (default):** + +```bash +docker exec -d ${CONTAINER} bash -c \ + "HIP_VISIBLE_DEVICES=${GPUS} ${LAUNCH_CMD} > ${EVAL_STATE_DIR}/vllm-server.log 2>&1" +``` + +**Host runtime variant** (when `runtime=host`): + +```bash +# Launch directly on the host; PID captured into the state dir for later signals. +( HIP_VISIBLE_DEVICES=${GPUS} ${LAUNCH_CMD} > "${EVAL_STATE_DIR}/vllm-server.log" 2>&1 ) & +echo "$!" > "${EVAL_STATE_DIR}/server.pid" +``` + +In host mode, every later step that says `docker exec ${CONTAINER} ` becomes just `` on the host; every "tail the in-container log" becomes `tail -F "${EVAL_STATE_DIR}/vllm-server.log"` (same path works in docker mode too, since the log file is bind-mounted). + +Write `$EVAL_STATE_DIR/server-runtime.md` immediately after launch (PID, port, container, image, full launch command). + +**Readiness wait (up to 300s):** use the poll template in `backends.md` §Readiness Poll Template — polls health every 5s with process liveness check, breaks immediately on process death instead of waiting 300s. + +**Background server monitor**: after the server is ready, launch the monitor via `Bash` with `run_in_background: true` and persist the returned `task_id`: + +```bash +# Bash, run_in_background: true. Capture task_id, then: +echo "server_monitor=${task_id}" >> "${EVAL_STATE_DIR}/monitor.pid" +# Both runtimes — log is bind-mounted into the container, so tail from host either way: +tail -F "${EVAL_STATE_DIR}/vllm-server.log" \ + | grep -E --line-buffered "Avg prompt throughput|Avg generation throughput|ERROR|error|OOM|Killed|CUDA|torch.OutOfMemoryError|engine is dead" +``` + +`TaskStop` every ID in `monitor.pid` at every exit path: Phase 5 retry (before each new launch), Phase 6 early-exit, Phase 7 cleanup. Skipping this leaks a background `tail -F` per session. + +### Phase 4 — Smoke test + quality gate + +Follow `backends.md` §Smoke Test Templates — verify model name via `/v1/models`, then send a simple inference request (chat or base). Both HTTP 200 and reasonable content are required. Either gate fails → enter Phase 5. + +### Phase 5 — Fix if needed + +Entered on Phase 3 launch failure or Phase 4 gate failure. Mode comes from `launch_cmd_source` in `current-eval.yaml`. + +**Reproduce mode** (`launch_cmd_source ∈ {quant_model_card, similar_quant_card}`) — the command is verified for this arch+quant combo, so do **not** auto-fix: + +1. `TaskStop` every ID in `$EVAL_STATE_DIR/monitor.pid` +2. Report: (a) core error line, (b) analysis from `troubleshooting.md`, (c) suggested fix +3. Wait for the user to choose explicitly: **`fall through to bringup`** (re-enter Phase 5 in Bringup mode) or **`abort`** (stop, offer Phase 7 cleanup). Do not silently fall through. Do not retry without instruction. + +**Bringup mode** (`launch_cmd_source ∈ {base_model_card, recipes, template}`) — run the fix cycle below. + +**Three rules (bringup only):** one fix at a time (no flag stacking); rerun the exact failing step (Phase 3 or 4, no skipping); max 3 attempts then stop and report the full log. + +**Fix cycle:** + +1. Get server log or traceback, locate the **core error line** +2. Match error signature against `troubleshooting.md` +3. Apply **one** minimal fix: + - env var / launch flag → apply directly + - source code / `config.json` → show full diff, wait for user approval + - no match → search `github.com/vllm-project/vllm/issues`; same approval rule for any source mod + - kernel / hardware-unsupported → stop and inform user +4. Rerun the exact failing step +5. Record `{attempt, error, fix, result}` in `current-eval.yaml` + +### Phase 6 — Eval + report + +Three steps, in order. Do not present the final report until all three are done. + +**Step 1 — Run eval.** Read `eval-frameworks.md` for command templates, task type classification, and `num_fewshot` lookup. Format the raw eval output as shown in the Outputs section above. + +**Step 2 — Accuracy comparison.** Use the `reference_scores` already collected in Phase 1. If Phase 1 found a reference, use it directly. If not, do a fallback lookup per `eval-frameworks.md` §Accuracy Comparison (paper → Open LLM Leaderboard → similar-scale model). Build the comparison table with the eval setting from the reference source. If the reference's eval setting differs from what was actually used, note the difference explicitly. No benchmark may be silently omitted — if no reference exists, write "N/A" with reason. + +**Step 3 — Anomaly detection.** Flag any delta > 5% absolute. See `eval-frameworks.md` §Anomaly detection for the diagnostic table. + +### Phase 7 — Post-eval cleanup + +After the evaluation report is produced, ask the user how to handle the running resources: + +0. **Stop background monitors** — for every line in `$EVAL_STATE_DIR/monitor.pid`, call `TaskStop` on the recorded `task_id`. Truncate the file after. +1. **Stop server?** + - Docker runtime: kill the inference process inside the container (`docker exec ${CONTAINER} pkill -f "vllm serve|sglang.launch_server|atom.entrypoints"`), then verify the port is released via `lsof -i :${PORT}`. + - Host runtime: stop the PID in `${EVAL_STATE_DIR}/server.pid` following the "Never kill other users' processes" rule above (L72-76) — verify the PID is alive and still our backend, then SIGTERM → wait → SIGKILL only if needed. Verify port with `lsof -i :${PORT}`. +2. **Remove container?** — `docker stop` + `docker rm`. Skip in host runtime. +3. **Rollback file edits?** — if `source_edits` is non-empty in `$EVAL_STATE_DIR/current-eval.yaml`, offer to restore each `.bak.` file: `cp .bak. ` +4. **Keep everything** — user may want to re-run or debug + +Always ask — never auto-kill. After cleanup (or skip), update `$EVAL_STATE_DIR/server-runtime.md`. + +## State Files + +`$EVAL_STATE_DIR` is set in Phase 0.5 to `/tmp/eval-state///` — unique per session and user. Every later phase **reads** the path from `/tmp/eval-state/.last-session-path-$(whoami)` rather than re-deriving from `whoami` + timestamp. + +Primary: `$EVAL_STATE_DIR/current-eval.yaml` — updated after each Phase. +Server: `$EVAL_STATE_DIR/server-runtime.md` — written at Phase 3 launch, kept current. + +**Fallback**: if `/tmp/eval-state/` is not writable, use `./.eval-state//` in the current working directory. Phase 0.5 preflight probes this and writes the resolved path as the first field of `current-eval.yaml` and into the `.last-session-path-` pointer. + +## References + +| File | When to read | +|------|-------------| +| `model-inspection.md` | Phase 0, for config parsing / GPU probing / TP math; §2 for all HF HTTP requests | +| `backends.md` | Phase 2-3, for docker setup + serving commands (all backends) | +| `troubleshooting.md` | Phase 5, on launch failure or smoke test failure | +| `eval-frameworks.md` | Phase 6, for eval commands, accuracy comparison lookup, and anomaly detection | +| `templates.md` | Phase 1 (pre-run confirmation) and Phase 6 (assembling `eval_report.md`) | diff --git a/skills/quark-torch-llm-eval/backends.md b/skills/quark-torch-llm-eval/backends.md new file mode 100644 index 0000000..55d2123 --- /dev/null +++ b/skills/quark-torch-llm-eval/backends.md @@ -0,0 +1,303 @@ +# Backends: Docker Setup + Serving Commands + +Docker container setup and serving commands for all three backends (vLLM, ATOM, SGLang) on AMD ROCm. + +## Docker Setup (Common ROCm Flags) + +**Flags required by ROCm (do not change):** + +- `--device=/dev/kfd --device=/dev/dri --device=/dev/mem`: expose GPUs +- `--cap-add=SYS_PTRACE --security-opt seccomp=unconfined`: permission set required by the ROCm kernel driver + +**Flags you may need to change:** + +- `--name`: container name. Recommend `eval---` (replace slashes with dashes, username from `whoami`). Example: `eval-amd-MiniMax-M2.7-mxfp4-jiaxwang-0427` +- `-v :`: mount the top-level directory of the user-provided `${MODEL_PATH}` at the same path inside the container (e.g. `MODEL_PATH=/scratch/.../foo` → `-v /scratch:/scratch`). Default to `-v /shareddata:/shareddata` only when no model path was given. +- `--shm-size`: vLLM multi-process communication needs shared memory. Default `64g`; MoE models may need `128g` + +**Container lifecycle:** + +```bash +docker pull # pull first to avoid blocking during run +docker run [...] # see backend-specific templates below +docker exec -it bash # enter container +``` + +**Private image authentication:** if `docker pull` fails with `unauthorized` / `authentication required` / `denied`, the image requires login. Detect the registry from the image name and guide the user: + +| Image name pattern | Registry | Login command | +|---|---|---| +| `/` (no dots) | Docker Hub | `docker login` | +| `/` (host has dots) | Private registry | `docker login ` | +| `.dkr.ecr..amazonaws.com/...` | AWS ECR | `aws ecr get-login-password --region \| docker login --username AWS --password-stdin .dkr.ecr..amazonaws.com` | + +After the user confirms they have logged in, retry `docker pull`. Do not proceed to `docker run` until the pull succeeds. + +Record `container_name` and `image` in `$EVAL_STATE_DIR/current-eval.yaml`. + +**Cleanup (after evaluation, on request only):** + +```bash +docker stop +docker rm +``` + +Do not clean up by default — the user may want to keep the container for reproduction or comparison. + +--- + +## vLLM + +**Default image:** `vllm/vllm-openai-rocm:latest` (see SKILL.md Phase 1 for full image priority chain) + +### Docker run template + +```bash +docker run -it -d \ + --entrypoint /bin/bash \ + --ipc=host \ + --shm-size=64g \ + --network=host \ + --name= \ + --privileged \ + --cap-add=CAP_SYS_ADMIN \ + --device=/dev/kfd \ + --device=/dev/dri \ + --device=/dev/mem \ + --cap-add=SYS_PTRACE \ + --security-opt seccomp=unconfined \ + -v /shareddata:/shareddata \ + -v ${EVAL_STATE_DIR}:${EVAL_STATE_DIR} \ + +``` + +The `-v ${EVAL_STATE_DIR}:${EVAL_STATE_DIR}` bind mount lets the in-container server write its log directly to the canonical state path (SKILL.md L134-140), so docker and host runtimes share the same log location with no symlink trick. + +### Serving command + +```bash +HIP_VISIBLE_DEVICES=${GPUS} vllm serve ${MODEL_PATH} --tensor-parallel-size ${TP} --port ${PORT:-8080} +``` + +--- + +## Cross-Backend Launch Adaptation Rules + +These apply to **every** backend (vLLM, ATOM, SGLang). They cover only adaptations YOU make on top of the source command (model card / Recipes / Cookbook). Preserve all other flags from the source command verbatim. + +| Change | Rule | +|--------|------| +| Replace base/quantized model HF id → local `${MODEL_PATH}` | **Always** — run the user's local copy, never re-download | +| Adjust tensor-parallel size (`--tensor-parallel-size` / `-tp` / `--tp`) | Use the source command's value if it fits; otherwise the smallest TP that satisfies `num_kv_heads % TP == 0` and `aggregate_free_vram >= model_size * 1.3` | +| Set `HIP_VISIBLE_DEVICES=${GPUS}` | **Always** — `${GPUS}` is the `hip_visible_devices` field from `current-eval.yaml` (already HIP indices, comma-joined). **Never** read `chosen_gpu_rocm` — those are rocm-smi indices and map to the wrong physical GPUs (`HIP_VISIBLE_DEVICES` expects HIP indices, derived from KFD node order). See `model-inspection.md` §4 for the rocm-smi → HIP mapping. | +| Set `--port ${PORT}` | If the source command omits a port, pick one (default 8080); if 8080 is busy, fall through 8081…8090 | + +**Reactive-only flags** — do not add preemptively, only when an error in `troubleshooting.md` instructs you to: + +- `--enforce-eager`, `--max-model-len`, `VLLM_ROCM_USE_AITER=1`, `--trust-remote-code`, `chat_template_kwargs`, `--reasoning-parser`, etc. + +The error-signature table in `troubleshooting.md` is the authoritative trigger for each of those. Do not stack flags speculatively. + +--- + +## ATOM + +ATOM (AiTer Optimized Model) is AMD's lightweight LLM inference engine built on [AITER](https://github.com/ROCm/aiter) kernels. Repository: [github.com/ROCm/ATOM](https://github.com/ROCm/ATOM). + +**Default image:** `rocm/atom-dev:latest` (nightly, recommended). Stable: `rocm/atom:latest`. (See SKILL.md Phase 1 for full image priority chain.) + +### Docker run template + +```bash +docker run -it -d \ + --entrypoint /bin/bash \ + --network=host \ + --name= \ + --device=/dev/kfd \ + --device=/dev/dri \ + --group-add video \ + --cap-add=SYS_PTRACE \ + --security-opt seccomp=unconfined \ + --shm-size=16G \ + --ulimit memlock=-1 \ + --ulimit stack=67108864 \ + -v /shareddata:/shareddata \ + -v ${EVAL_STATE_DIR}:${EVAL_STATE_DIR} \ + rocm/atom-dev:latest +``` + +Note: ATOM's docker template differs from vLLM — uses `--group-add video`, `--ulimit memlock=-1`, `--ulimit stack=67108864` instead of `--privileged`. + +### Serving commands + +**Basic:** + +```bash +python -m atom.entrypoints.openai_server --model ${MODEL_PATH} --kv_cache_dtype fp8 -tp ${TP} --port ${PORT:-8080} +``` + +**With MTP speculative decoding (~60% throughput improvement):** + +```bash +python -m atom.entrypoints.openai_server --model ${MODEL_PATH} --kv_cache_dtype fp8 -tp ${TP} --port ${PORT:-8080} \ + --method mtp --num-speculative-tokens 3 +``` + +### Key parameters + +- `--kv_cache_dtype fp8`: always recommended for memory efficiency +- `-tp `: tensor parallelism +- `--method mtp --num-speculative-tokens 3`: MTP speculative decoding (best throughput/latency tradeoff) +- `--num-speculative-tokens 1`: more conservative MTP with lower overhead +- `--trust-remote-code`: required for some models (e.g., Kimi-K2) + +### ATOM as vLLM plugin + +ATOM can run as a vLLM out-of-tree plugin. When `atom` is installed alongside `vllm`, `vllm serve` automatically loads ATOM's optimized kernels. + +```bash +pip install amd-aiter +git clone https://github.com/ROCm/ATOM.git && pip install ./ATOM +# Then just use vllm serve as normal — ATOM hooks are auto-activated +vllm serve ${MODEL_PATH} --tensor-parallel-size ${TP} +``` + +To disable the plugin: `export ATOM_DISABLE_VLLM_PLUGIN=1` + +### ATOM Recipes + +Per-model deployment guides live in the `recipes/` directory of the [ROCm/ATOM](https://github.com/ROCm/ATOM) repo. **List the current set at runtime — do not rely on a hardcoded list (it goes stale within weeks).** + +```bash +# List available recipe names +curl -fsSL https://api.github.com/repos/ROCm/ATOM/contents/recipes \ + | python3 -c "import sys,json; [print(i['name']) for i in json.load(sys.stdin) if i['name'].endswith('.md')]" + +# Fetch one +curl -fsSL https://raw.githubusercontent.com/ROCm/ATOM/main/recipes/${RECIPE_NAME} +``` + +If a recipe matching the model family is found (e.g., `DeepSeek-R1.md` for a DeepSeek-V3 quant), use it as the Phase 1 launch-command source. + +### Tips + +- Set `AITER_LOG_LEVEL=WARNING` to suppress aiter kernel log noise +- Clear compile cache before restarting: `rm -rf /root/.cache/atom/*` +- First-time execution takes ~10 minutes for model compilation +- Performance dashboard: [rocm.github.io/ATOM/benchmark-dashboard](https://rocm.github.io/ATOM/benchmark-dashboard/) + +### Quantization support + +ATOM auto-detects quantization configs from HuggingFace model configs: FP8, MXFP4, INT8, INT4. + +--- + +## SGLang + +**Default image:** `lmsysorg/sglang:v0.5.11-rocm720-mi35x` (See SKILL.md Phase 1 for full image priority chain.) + +> **Status — partial support.** Launch templates and the readiness/smoke flow apply. `troubleshooting.md` currently covers vLLM-only error signatures; SGLang-specific failures fall through to "search GitHub Issues". SGLang-specific troubleshooting will be added in a follow-up. + +### Serving command + +```bash +python -m sglang.launch_server --model-path ${MODEL_PATH} --tp ${TP} --port ${PORT:-8080} +``` + +### Docker run template + +Use the common ROCm docker template (same shape as the vLLM section; replace the `` line with the SGLang default above). SGLang Cookbook: `https://lmsysorg.mintlify.app/cookbook/` + +--- + +## Host (No-Docker) Launch Commands + +Use when `runtime=host` is selected in Phase 0.5. ATOM is docker-only and has no host variant. + +**vLLM (host):** + +```bash +HIP_VISIBLE_DEVICES=${GPUS} vllm serve ${MODEL_PATH} --tensor-parallel-size ${TP} --port ${PORT:-8080} +``` + +**SGLang (host):** + +```bash +HIP_VISIBLE_DEVICES=${GPUS} python -m sglang.launch_server --model-path ${MODEL_PATH} --tp ${TP} --port ${PORT:-8080} +``` + +Wrap either of those in the host-mode launch template from `SKILL.md` Phase 3 — the `( ... > "${EVAL_STATE_DIR}/vllm-server.log" 2>&1 ) &` form that captures the PID into `${EVAL_STATE_DIR}/server.pid`. The Cross-Backend Launch Adaptation Rules and reactive-only flag policy above apply identically — only the `docker run` / `docker exec` plumbing drops out. + +## Readiness Poll Template + +Poll health endpoint every 5s with process liveness check. If the server process exits early (bad arguments, import error), break immediately instead of waiting 300s. + +**Docker runtime:** + +```bash +# Required vars: CONTAINER, PORT (export before running) +: "${CONTAINER:?must export CONTAINER}" +: "${PORT:?must export PORT}" +for i in $(seq 1 60); do + if ! docker exec "${CONTAINER}" pgrep -f "vllm serve|sglang.launch_server|atom.entrypoints" >/dev/null 2>&1; then + echo "PROCESS DEAD after $((i*5)) seconds" + tail -30 "${EVAL_STATE_DIR}/vllm-server.log" + break + fi + if docker exec "${CONTAINER}" curl -sf "http://localhost:${PORT}/health" >/dev/null 2>&1; then + echo "READY after $((i*5)) seconds"; break + fi + sleep 5 +done +``` + +**Host runtime variant:** + +```bash +# Required vars: PORT, EVAL_STATE_DIR (export before running) +: "${PORT:?must export PORT}" +: "${EVAL_STATE_DIR:?must export EVAL_STATE_DIR}" +SERVER_PID=$(cat "${EVAL_STATE_DIR}/server.pid") +for i in $(seq 1 60); do + if ! kill -0 "${SERVER_PID}" 2>/dev/null; then + echo "PROCESS DEAD after $((i*5)) seconds" + tail -30 "${EVAL_STATE_DIR}/vllm-server.log" + break + fi + if curl -sf "http://localhost:${PORT}/health" >/dev/null 2>&1; then + echo "READY after $((i*5)) seconds"; break + fi + sleep 5 +done +``` + +Process dead → read log tail, enter Phase 5. 300s timeout → read log tail, enter Phase 5. + +## Smoke Test Templates + +Required vars: `PORT`, `MODEL_PATH` (export before running). + +**Step 0 — verify model name:** `curl -s "http://localhost:${PORT}/v1/models"` — confirm served model name matches what you will use in eval commands. + +**Chat models:** + +```bash +curl "http://localhost:${PORT}/v1/chat/completions" \ + -H "Content-Type: application/json" \ + -d "{\"model\":\"${MODEL_PATH}\",\"messages\":[{\"role\":\"user\",\"content\":\"What is the capital of China?\"}],\"max_tokens\":100}" +``` + +**Base models:** + +```bash +curl "http://localhost:${PORT}/v1/completions" \ + -H "Content-Type: application/json" \ + -d "{\"model\":\"${MODEL_PATH}\",\"prompt\":\"The capital of China is\",\"max_tokens\":50}" +``` + +**Both gates must pass:** + +1. HTTP 200 + JSON has `choices[0].message.content` (chat) or `choices[0].text` (completions) +2. Content is reasonable: not empty, not single-character repetition, not gibberish, topically relevant + +Either gate fails → enter Phase 5. diff --git a/skills/quark-torch-llm-eval/eval-frameworks.md b/skills/quark-torch-llm-eval/eval-frameworks.md new file mode 100644 index 0000000..5035a6b --- /dev/null +++ b/skills/quark-torch-llm-eval/eval-frameworks.md @@ -0,0 +1,453 @@ +# Evaluation Frameworks + +Phase 6 of the main workflow calls this file. + +## Model Type Detection + +See `model-inspection.md` §3 for the canonical detection logic. The result is recorded as `model_type_for_eval: chat | base` in the state file. + +## Benchmark → Framework Mapping + +| Benchmark | Framework | Notes | +|---|---|---| +| `gsm8k` / `mmlu` / `hellaswag` / `arc_*` / `truthfulqa` / `winogrande` | **lm-eval** | Classic academic benchmarks | +| `mmlu_pro` / `ifeval` / `bbh` | **lm-eval** | Supported in lm-eval 0.4.3+ | +| `aime24` / `math_500` / `gpqa:diamond` | **lighteval** or **evalscope** | Math/reasoning benchmarks | + +**Decision rule for benchmarks not in the table:** + +1. Check model card / Recipes for a recommended framework → use it if found +2. No recommendation → run `lm_eval --tasks list 2>/dev/null | grep -i ` in the container to check lm-eval support +3. lm-eval does not support it → check `lighteval` task list (`python -c "from lighteval.tasks.registry import Registry; ..."` or docs) +4. Not in lighteval → check `evalscope` +5. None of the three support it → inform the user and ask them to provide an evaluation script + +## Eval Command Discovery Chain + +See SKILL.md Phase 1 for the full priority chain (quantized model card → sibling quant variant → base model card → Recipes → framework repos → InferenceX → generic templates). Each item (`launch_cmd`, `eval_cmd`) walks the chain independently. + +**Model card eval command takes precedence over generic templates.** If a model card (from any level in the chain) provides a specific eval command with verified accuracy, use that command regardless of the chat/base template mapping below. For example, some chat models achieve better benchmark accuracy with `local-completions` + few-shot prompting (bypassing the chat template) — the model card author tested this and confirmed the accuracy. Do not override their choice with the generic chat→`local-chat-completions` mapping. + +Record the final command and its source in the state file. + +## Task Request Type: loglikelihood vs generation + +lm-eval tasks use one of two request types. The request type determines which API endpoint and model wrapper to use: + +| Request type | Description | API endpoint | lm-eval model type | +|---|---|---|---| +| `generate_until` | Model generates free-form text (e.g., gsm8k, humaneval) | chat/completions or completions | `local-chat-completions` (chat) or `local-completions` (base) | +| `loglikelihood` | Model scores each answer option by probability (e.g., mmlu, mmlu_pro, hellaswag, arc, winogrande) | completions only | `local-completions` only | + +**`local-chat-completions` does NOT support loglikelihood.** If a loglikelihood task is run with `local-chat-completions`, lm-eval raises `NotImplementedError: Loglikelihood is not supported for chat completions`. + +**How to determine request type before running** (export `CONTAINER` and `TASK_NAME` first; written to a temp file per the §2 execution rule in `model-inspection.md`): + +```bash +cat > /tmp/check_task_type.py <<'PYEOF' +import os +from lm_eval.tasks import TaskManager +tm = TaskManager() +cfg = tm.get_task_config(os.environ['TASK_NAME']) +print(f"output_type: {cfg.get('output_type', 'unknown')}") +PYEOF +docker cp /tmp/check_task_type.py "${CONTAINER}":/tmp/check_task_type.py +docker exec -e TASK_NAME="${TASK_NAME}" "${CONTAINER}" python3 /tmp/check_task_type.py +``` + +If the above fails or returns `unknown`, check common task types: + +- **loglikelihood tasks**: `mmlu`, `mmlu_pro`, `leaderboard_mmlu_pro`, `hellaswag`, `arc_easy`, `arc_challenge`, `winogrande`, `truthfulqa_mc2`, `piqa`, `boolq`, `openbookqa` +- **generate_until tasks**: `gsm8k`, `math`, `humaneval`, `mbpp`, `aime24`, `math_500`, `gpqa`, `ifeval`, `bbh` + +**Scope**: this decision rule only applies when constructing eval commands from generic templates (`eval_cmd_source: template`). When `eval_settings` have already been determined by a higher-priority source (model card, recipes, or paper), use those settings directly — do not re-derive eval_mode or num_fewshot from the rules below. + +**Decision rule:** + +1. If task is `loglikelihood` → always use `local-completions` + completions API, regardless of whether the model is chat or base. **Must add `max_length=` to model_args** — lm-eval defaults to 2048 which truncates few-shot prompts for long benchmarks (mmlu_pro, mmlu, etc.), causing severe accuracy loss. +2. If task is `generate_until` → use `local-chat-completions` for chat models, `local-completions` for base models + +This check MUST happen in Phase 6 before constructing the eval command. Record `task_request_type: loglikelihood | generate_until` in the state file. + +## Thinking Model Handling + +Thinking/reasoning models (Qwen3, QwQ, DeepSeek-R1, etc.) output `...` blocks before the actual answer. This causes two problems for lm-eval: + +1. **generate_until tasks** (gsm8k, math, etc.): lm-eval's regex extractors (e.g., `flexible-extract` looking for `#### `) fail because the `` block contains intermediate calculations that confuse the pattern matcher, resulting in near-zero accuracy. +2. **loglikelihood tasks** (mmlu, mmlu_pro, etc.): not directly affected by thinking output (loglikelihood doesn't generate), but may still score poorly if the model's probability estimates are distorted by its tendency to "think first." + +**Detection:** In Phase 0, check if the model is a thinking model. The check must gate on the model **type**, not just the family — Qwen3 base checkpoints have no chat template and the soft `/no_think` switch is a no-op on them, so applying it silently does nothing useful and can confuse downstream prompt accounting. Required vars: `MODEL_NAME`, `MODEL_TYPE_FOR_EVAL` (from `model-inspection.md` §3). + +```python +is_thinking_model = ( + MODEL_TYPE_FOR_EVAL == 'chat' + and ( + re.search(r'(QwQ|-R1\b)', MODEL_NAME, re.I) # always thinking + or re.search(r'Qwen3.*-(Instruct|Chat|Thinking)', MODEL_NAME, re.I) # Qwen3 instruct/chat/thinking variants only + or model_card_mentions_any(['thinking mode', 'enable_thinking', 'reasoning mode']) + ) +) +``` + +Record as `is_thinking_model: true|false` in the state file. Do NOT set true for Qwen3 base checkpoints (e.g., `Qwen3-8B-Base`) or for any model whose `model_type_for_eval` is `base`. + +**Fix for generate_until tasks — disable thinking mode:** + +The preferred approach depends on what the vLLM version and model support: + +1. **Best: `--system_instruction "/no_think"`** — add to the lm-eval command. This inserts `/no_think` as the system message, which Qwen3 models recognize as a soft switch to disable thinking. The model still outputs empty `\n\n` tags but no reasoning content, so extractors work correctly. + + ```bash + lm_eval --model local-chat-completions --apply_chat_template \ + --system_instruction "/no_think" \ + --tasks gsm8k ... + ``` + +2. **Alternative: `chat_template_kwargs`** — pass `{"enable_thinking": false}` via the vLLM API's `chat_template_kwargs` parameter. This completely suppresses `` tags. However, lm-eval may not support passing this through `gen_kwargs` or `extra_body` in all versions. + +3. **If vLLM supports `--enable-reasoning`** — launch with `--enable-reasoning --reasoning-parser deepseek_r1` so vLLM separates thinking content into `reasoning_content` field, keeping `content` clean. But this flag is not available in all vLLM builds. + +**Note:** Disabling thinking may reduce accuracy on reasoning-heavy benchmarks (math, aime). If the model card reports accuracy with thinking enabled, the results with `/no_think` may be lower. Document which mode was used in the eval report. + +## Eval Command Type Classification + +Before executing, classify the `eval_cmd` collected in Phase 1: + +| Pattern | Type | Action | +|---------|------|--------| +| `python vllm/tests/evals/...` or `python tests/evals/...` | **vllm-script** | Download the single script from vLLM GitHub, run directly | +| `lm_eval --model ...` | **lm-eval** | Install framework if missing, run | +| `lighteval ...` | **lighteval** | Install framework if missing, run | +| `evalscope ...` | **evalscope** | Install framework if missing, run | +| Other `python