From 8f2b159258508f7c202b35acadf7cd1ec658c201 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 3 Mar 2026 10:59:27 -0700 Subject: [PATCH 1/2] add feedback collection skill and list output hint New meta-skill guides agents to collect structured feedback after using SKILL.md files. The `list` command in both CLIs now shows a feedback hint pointing agents to the instructions and submit command. Co-Authored-By: Claude Opus 4.6 --- .../intent/meta/feedback-collection/SKILL.md | 151 ++++++++++++++++++ packages/intent/src/cli.ts | 6 + packages/intent/src/init.ts | 2 + packages/intent/src/intent-library.ts | 6 + 4 files changed, 165 insertions(+) create mode 100644 packages/intent/meta/feedback-collection/SKILL.md diff --git a/packages/intent/meta/feedback-collection/SKILL.md b/packages/intent/meta/feedback-collection/SKILL.md new file mode 100644 index 00000000..1d212bc9 --- /dev/null +++ b/packages/intent/meta/feedback-collection/SKILL.md @@ -0,0 +1,151 @@ +--- +name: skill-feedback-collection +description: > + Collect structured feedback about skill usage after completing a coding task. + Activate at the end of any session where one or more SKILL.md files were + loaded. Captures agent signals (gaps, errors, corrections, human interventions) + and brief human input, then submits via the intent feedback CLI. +metadata: + version: '1.0' + category: meta-tooling + output_artifacts: + - intent-feedback.json +--- + +# Skill Feedback Collection + +Run this at the end of any session where you loaded one or more SKILL.md files. +The goal is to capture what worked, what didn't, and what was missing — so skill +maintainers can improve future versions. + +--- + +## Phase 1 — Automated Signal Collection + +Review your own session transcript. No human interaction needed yet. + +### 1a: Gap detection + +Identify moments where the skill was silent and you had to bridge the gap +yourself — via code reading, search, trial-and-error, or general knowledge. + +For each gap, note: +- What you needed to do +- What the skill should have told you +- How you solved it (code reading, web search, guessing) + +### 1b: Error/correction tracking + +Identify moments where the skill prescribed an approach that produced an error. + +For each error, note: +- What the skill said to do +- The error or incorrect behavior that resulted +- The fix you applied + +### 1c: Human intervention events + +Identify moments where the human clarified, corrected, or overrode your approach. + +For each intervention, note: +- What you were doing when the human intervened +- What the human said or changed +- Whether the skill could have prevented this + +### 1d: Step duration anomalies + +Identify steps that consumed disproportionate effort compared to their apparent +complexity. These signal that the skill should provide a template, snippet, or +more detailed guidance. + +--- + +## Phase 2 — Human Interview + +Ask the human up to 4 questions. Keep it brief — skip questions if the session +already provided clear answers. Respect if they decline. + +1. "Was anything unclear about what was happening during the task?" +2. "Did anything feel frustrating or take longer than expected?" +3. "Were you uncertain about the output quality at any point?" +4. "Anything you'd want done differently next time?" + +Derive `userRating` from overall sentiment: +- Mostly positive → `good` +- Mixed signals → `mixed` +- Mostly negative → `bad` + +If the human gives an explicit rating, use that instead. + +--- + +## Phase 3 — Build the Payload + +Construct one JSON payload per skill used. The schema must match exactly: + +```json +{ + "skill": "", + "package": "", + "skillVersion": "", + "task": "", + "whatWorked": "", + "whatFailed": "", + "missing": "", + "selfCorrections": "", + "userRating": "good | mixed | bad", + "userComments": "" +} +``` + +### Field derivation guide + +| Field | Source | +|-------|--------| +| `skill` | Frontmatter `name` field of the SKILL.md you loaded | +| `package` | The npm package the skill lives in (e.g. `@tanstack/query-intent`) | +| `skillVersion` | Frontmatter `metadata.version` or `library_version` | +| `task` | Summarize the human's original request in one sentence | +| `whatWorked` | List skill sections/patterns that were correct and useful | +| `whatFailed` | From 1b — skill instructions that produced errors | +| `missing` | From 1a — gaps where the skill was silent | +| `selfCorrections` | From 1b fixes + 1c human interventions, combined | +| `userRating` | From Phase 2 sentiment analysis or explicit rating | +| `userComments` | From Phase 2 answers, keep brief | + +### Example + +```json +{ + "skill": "tanstack-query/core", + "package": "@anthropic/tanstack-query-intent", + "skillVersion": "1.0", + "task": "Add optimistic updates to a mutation with rollback on error", + "whatWorked": "Setup pattern was correct. onMutate/onError/onSettled lifecycle was accurate.", + "whatFailed": "Cache key format used array syntax that doesn't match v5 — had to switch to queryOptions pattern.", + "missing": "No guidance on TypeScript generics for mutation variables. Had to read source.", + "selfCorrections": "Fixed cache key format. Human corrected the rollback type to include undefined.", + "userRating": "mixed", + "userComments": "The optimistic update pattern was helpful but the cache key thing wasted 10 minutes." +} +``` + +If multiple skills were loaded, produce one payload per skill. Only include +skills that were actually used during the session — skip any that were loaded +but never referenced. + +--- + +## Phase 4 — Submit + +1. Save the payload to `intent-feedback.json` in the project root. + If multiple skills, save as a JSON array. + +2. Submit: + ```bash + npx intent feedback --submit --file intent-feedback.json + ``` + +3. If the submission succeeds, delete `intent-feedback.json`. + +4. If it fails, tell the human and leave the file for manual retry. diff --git a/packages/intent/src/cli.ts b/packages/intent/src/cli.ts index 16d804ac..d6389569 100644 --- a/packages/intent/src/cli.ts +++ b/packages/intent/src/cli.ts @@ -167,6 +167,12 @@ async function cmdList(args: string[]): Promise { console.log() } + console.log(`Feedback:`) + console.log(` After completing your task, collect feedback on skill usage.`) + console.log(` Instructions: node_modules/@tanstack/intent/meta/feedback-collection/SKILL.md`) + console.log(` Submit: npx intent feedback --submit --file intent-feedback.json`) + console.log() + if (result.warnings.length > 0) { console.log(`Warnings:`) for (const w of result.warnings) console.log(` ⚠ ${w}`) diff --git a/packages/intent/src/init.ts b/packages/intent/src/init.ts index 95056c11..24a4e953 100644 --- a/packages/intent/src/init.ts +++ b/packages/intent/src/init.ts @@ -20,6 +20,8 @@ const INTENT_BLOCK = `## Intent Skills This project uses TanStack Intent. Run \`npx intent list\` to discover available AI coding skills. Before working with a library that has skills, read the relevant SKILL.md file at the path shown in the list output. +After completing a task that used skills, collect and submit feedback +following the instructions in the list output. ` const DEFAULT_CONFIG: IntentProjectConfig = { diff --git a/packages/intent/src/intent-library.ts b/packages/intent/src/intent-library.ts index 750ec2e0..78568cb4 100644 --- a/packages/intent/src/intent-library.ts +++ b/packages/intent/src/intent-library.ts @@ -145,6 +145,12 @@ async function cmdList(): Promise { console.log() } + console.log(`Feedback:`) + console.log(` After completing your task, collect feedback on skill usage.`) + console.log(` Instructions: node_modules/@tanstack/intent/meta/feedback-collection/SKILL.md`) + console.log(` Submit: npx intent feedback --submit --file intent-feedback.json`) + console.log() + if (result.warnings.length > 0) { console.log(`Warnings:`) for (const w of result.warnings) console.log(` ⚠ ${w}`) From 951399d700469d4943e91456cb6a607c82a37074 Mon Sep 17 00:00:00 2001 From: "autofix-ci[bot]" <114827586+autofix-ci[bot]@users.noreply.github.com> Date: Tue, 3 Mar 2026 18:00:19 +0000 Subject: [PATCH 2/2] ci: apply automated fixes --- docs/intent/functions/detectAgentConfigs.md | 2 +- docs/intent/functions/hasIntentBlock.md | 2 +- docs/intent/functions/injectIntentBlock.md | 2 +- docs/intent/functions/readProjectConfig.md | 2 +- docs/intent/functions/runInit.md | 2 +- docs/intent/functions/writeProjectConfig.md | 2 +- .../intent/meta/feedback-collection/SKILL.md | 29 +++++++++++-------- packages/intent/src/cli.ts | 8 +++-- packages/intent/src/intent-library.ts | 8 +++-- 9 files changed, 35 insertions(+), 22 deletions(-) diff --git a/docs/intent/functions/detectAgentConfigs.md b/docs/intent/functions/detectAgentConfigs.md index 43c70d08..10d19460 100644 --- a/docs/intent/functions/detectAgentConfigs.md +++ b/docs/intent/functions/detectAgentConfigs.md @@ -9,7 +9,7 @@ title: detectAgentConfigs function detectAgentConfigs(root): string[]; ``` -Defined in: [init.ts:35](https://github.com/TanStack/intent/blob/main/packages/intent/src/init.ts#L35) +Defined in: [init.ts:37](https://github.com/TanStack/intent/blob/main/packages/intent/src/init.ts#L37) ## Parameters diff --git a/docs/intent/functions/hasIntentBlock.md b/docs/intent/functions/hasIntentBlock.md index 8ca116f0..8b25a0d8 100644 --- a/docs/intent/functions/hasIntentBlock.md +++ b/docs/intent/functions/hasIntentBlock.md @@ -9,7 +9,7 @@ title: hasIntentBlock function hasIntentBlock(filePath): boolean; ``` -Defined in: [init.ts:45](https://github.com/TanStack/intent/blob/main/packages/intent/src/init.ts#L45) +Defined in: [init.ts:47](https://github.com/TanStack/intent/blob/main/packages/intent/src/init.ts#L47) ## Parameters diff --git a/docs/intent/functions/injectIntentBlock.md b/docs/intent/functions/injectIntentBlock.md index d2cd7f90..a25a9fc7 100644 --- a/docs/intent/functions/injectIntentBlock.md +++ b/docs/intent/functions/injectIntentBlock.md @@ -9,7 +9,7 @@ title: injectIntentBlock function injectIntentBlock(filePath): boolean; ``` -Defined in: [init.ts:54](https://github.com/TanStack/intent/blob/main/packages/intent/src/init.ts#L54) +Defined in: [init.ts:56](https://github.com/TanStack/intent/blob/main/packages/intent/src/init.ts#L56) ## Parameters diff --git a/docs/intent/functions/readProjectConfig.md b/docs/intent/functions/readProjectConfig.md index 3765b783..a83042cc 100644 --- a/docs/intent/functions/readProjectConfig.md +++ b/docs/intent/functions/readProjectConfig.md @@ -9,7 +9,7 @@ title: readProjectConfig function readProjectConfig(root): IntentProjectConfig; ``` -Defined in: [init.ts:83](https://github.com/TanStack/intent/blob/main/packages/intent/src/init.ts#L83) +Defined in: [init.ts:85](https://github.com/TanStack/intent/blob/main/packages/intent/src/init.ts#L85) ## Parameters diff --git a/docs/intent/functions/runInit.md b/docs/intent/functions/runInit.md index 08e3e06a..9f69c0b2 100644 --- a/docs/intent/functions/runInit.md +++ b/docs/intent/functions/runInit.md @@ -9,7 +9,7 @@ title: runInit function runInit(root): InitResult; ``` -Defined in: [init.ts:103](https://github.com/TanStack/intent/blob/main/packages/intent/src/init.ts#L103) +Defined in: [init.ts:105](https://github.com/TanStack/intent/blob/main/packages/intent/src/init.ts#L105) ## Parameters diff --git a/docs/intent/functions/writeProjectConfig.md b/docs/intent/functions/writeProjectConfig.md index 85e00719..b95c50d5 100644 --- a/docs/intent/functions/writeProjectConfig.md +++ b/docs/intent/functions/writeProjectConfig.md @@ -9,7 +9,7 @@ title: writeProjectConfig function writeProjectConfig(root): string; ``` -Defined in: [init.ts:75](https://github.com/TanStack/intent/blob/main/packages/intent/src/init.ts#L75) +Defined in: [init.ts:77](https://github.com/TanStack/intent/blob/main/packages/intent/src/init.ts#L77) ## Parameters diff --git a/packages/intent/meta/feedback-collection/SKILL.md b/packages/intent/meta/feedback-collection/SKILL.md index 1d212bc9..fb7971d7 100644 --- a/packages/intent/meta/feedback-collection/SKILL.md +++ b/packages/intent/meta/feedback-collection/SKILL.md @@ -30,6 +30,7 @@ Identify moments where the skill was silent and you had to bridge the gap yourself — via code reading, search, trial-and-error, or general knowledge. For each gap, note: + - What you needed to do - What the skill should have told you - How you solved it (code reading, web search, guessing) @@ -39,6 +40,7 @@ For each gap, note: Identify moments where the skill prescribed an approach that produced an error. For each error, note: + - What the skill said to do - The error or incorrect behavior that resulted - The fix you applied @@ -48,6 +50,7 @@ For each error, note: Identify moments where the human clarified, corrected, or overrode your approach. For each intervention, note: + - What you were doing when the human intervened - What the human said or changed - Whether the skill could have prevented this @@ -71,6 +74,7 @@ already provided clear answers. Respect if they decline. 4. "Anything you'd want done differently next time?" Derive `userRating` from overall sentiment: + - Mostly positive → `good` - Mixed signals → `mixed` - Mostly negative → `bad` @@ -100,18 +104,18 @@ Construct one JSON payload per skill used. The schema must match exactly: ### Field derivation guide -| Field | Source | -|-------|--------| -| `skill` | Frontmatter `name` field of the SKILL.md you loaded | -| `package` | The npm package the skill lives in (e.g. `@tanstack/query-intent`) | -| `skillVersion` | Frontmatter `metadata.version` or `library_version` | -| `task` | Summarize the human's original request in one sentence | -| `whatWorked` | List skill sections/patterns that were correct and useful | -| `whatFailed` | From 1b — skill instructions that produced errors | -| `missing` | From 1a — gaps where the skill was silent | -| `selfCorrections` | From 1b fixes + 1c human interventions, combined | -| `userRating` | From Phase 2 sentiment analysis or explicit rating | -| `userComments` | From Phase 2 answers, keep brief | +| Field | Source | +| ----------------- | ------------------------------------------------------------------ | +| `skill` | Frontmatter `name` field of the SKILL.md you loaded | +| `package` | The npm package the skill lives in (e.g. `@tanstack/query-intent`) | +| `skillVersion` | Frontmatter `metadata.version` or `library_version` | +| `task` | Summarize the human's original request in one sentence | +| `whatWorked` | List skill sections/patterns that were correct and useful | +| `whatFailed` | From 1b — skill instructions that produced errors | +| `missing` | From 1a — gaps where the skill was silent | +| `selfCorrections` | From 1b fixes + 1c human interventions, combined | +| `userRating` | From Phase 2 sentiment analysis or explicit rating | +| `userComments` | From Phase 2 answers, keep brief | ### Example @@ -142,6 +146,7 @@ but never referenced. If multiple skills, save as a JSON array. 2. Submit: + ```bash npx intent feedback --submit --file intent-feedback.json ``` diff --git a/packages/intent/src/cli.ts b/packages/intent/src/cli.ts index d6389569..e0f972f8 100644 --- a/packages/intent/src/cli.ts +++ b/packages/intent/src/cli.ts @@ -169,8 +169,12 @@ async function cmdList(args: string[]): Promise { console.log(`Feedback:`) console.log(` After completing your task, collect feedback on skill usage.`) - console.log(` Instructions: node_modules/@tanstack/intent/meta/feedback-collection/SKILL.md`) - console.log(` Submit: npx intent feedback --submit --file intent-feedback.json`) + console.log( + ` Instructions: node_modules/@tanstack/intent/meta/feedback-collection/SKILL.md`, + ) + console.log( + ` Submit: npx intent feedback --submit --file intent-feedback.json`, + ) console.log() if (result.warnings.length > 0) { diff --git a/packages/intent/src/intent-library.ts b/packages/intent/src/intent-library.ts index 78568cb4..3d83b766 100644 --- a/packages/intent/src/intent-library.ts +++ b/packages/intent/src/intent-library.ts @@ -147,8 +147,12 @@ async function cmdList(): Promise { console.log(`Feedback:`) console.log(` After completing your task, collect feedback on skill usage.`) - console.log(` Instructions: node_modules/@tanstack/intent/meta/feedback-collection/SKILL.md`) - console.log(` Submit: npx intent feedback --submit --file intent-feedback.json`) + console.log( + ` Instructions: node_modules/@tanstack/intent/meta/feedback-collection/SKILL.md`, + ) + console.log( + ` Submit: npx intent feedback --submit --file intent-feedback.json`, + ) console.log() if (result.warnings.length > 0) {