From 26a5d914154b92b6a5d60eff287217be37bc0b7c Mon Sep 17 00:00:00 2001
From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Date: Fri, 24 Apr 2026 00:26:28 +0000
Subject: [PATCH 1/6] Add write commands to generated CLI and help overrides

Co-Authored-By: Aaryan Divate <divateaaryan@gmail.com>
---
 scripts/help_overrides.py              | 251 +++++++++++++++++++++
 src/judgment_cli/generated_commands.py | 301 +++++++++++++++++++++++++
 2 files changed, 552 insertions(+)

diff --git a/scripts/help_overrides.py b/scripts/help_overrides.py
index 905f52e..8b4b0fc 100644
--- a/scripts/help_overrides.py
+++ b/scripts/help_overrides.py
@@ -80,6 +80,93 @@
 """
 
 
+_AUTOMATION_CONDITIONS_HELP = """\
+JSON array of rule conditions. Each condition compares a behavior score or metric to a threshold. Items are ANDed or ORed together based on --combine-type ("all" vs "any").
+
+\b
+Condition shape:
+  {"metric": {"type":<metric_type>, ...}, "operator": ">=|<=|>|<|==|!=", "threshold": <number>}
+
+\b
+Supported metric types:
+  behavior       {"type":"behavior","behavior_id":"<uuid>"}
+  latency        {"type":"latency"}           (milliseconds)
+  cost           {"type":"cost"}              (USD)
+  token_count    {"type":"token_count"}
+
+Example: --conditions '[{"metric":{"type":"behavior","behavior_id":"<uuid>"},"operator":">=","threshold":0.8}]'\
+"""
+
+_AUTOMATION_ACTIONS_HELP = """\
+JSON object describing notification actions to fire when the automation triggers. Omit to store the automation with no actions.
+
+\b
+Shape: {"slack":[{"channel":"<channel>","webhook_url":"<url>"}], "pagerduty":[{"integration_key":"<key>"}], "email":[{"to":"<addr>"}]}
+
+Example: --actions '{"slack":[{"channel":"#alerts","webhook_url":"https://hooks.slack.com/..."}]}'\
+"""
+
+_AUTOMATION_TRIGGER_FREQUENCY_HELP = """\
+JSON 3-tuple [count, period, unit] describing the rate-limit window. Omit to disable frequency limiting.
+
+\b
+Shape: [<max_trigger_count:int>, <period:int>, <unit:"seconds"|"minutes"|"hours"|"days">]
+
+Example: --trigger-frequency '[5, 1, "hours"]'  (max 5 triggers per 1 hour)\
+"""
+
+_AUTOMATION_COOLDOWN_HELP = """\
+JSON 2-tuple [period, unit] describing the minimum wait between triggers.
+
+\b
+Shape: [<period:int>, <unit:"seconds"|"minutes"|"hours"|"days">]
+
+Example: --cooldown-period '[15, "minutes"]'  (at least 15 min between triggers)\
+"""
+
+_BEHAVIOR_ADVANCED_SETTINGS_HELP = """\
+JSON object overriding the judge's online evaluation configuration. If omitted, defaults are applied.
+
+\b
+Shape:
+  {
+    "online_evaluation_mode": "always" | "sampled" | "off",
+    "online_sampling_rate": <number 0-1>,
+    "online_span_triggers": [{"span_name":"<name>","attribute_filters":[...]}]?,
+    "online_session_scoring": <bool>?
+  }
+
+Example: --advanced-settings '{"online_evaluation_mode":"sampled","online_sampling_rate":0.1}'\
+"""
+
+_BEHAVIOR_CATEGORY_IDS_HELP = (
+    "UUID of a category to attach the behavior to. Repeat the flag to attach "
+    "multiple categories: --category-ids <uuid1> --category-ids <uuid2>."
+)
+
+_CLASSIFIER_OPTIONS_HELP = """\
+JSON array of the allowed output categories the classifier judge can return. Must contain at least one option.
+
+\b
+Shape:
+  [
+    {"name":"<label>", "description":"<optional human description>", "category_ids":["<uuid>", ...]},
+    ...
+  ]
+
+Example: --options '[{"name":"Good","description":"helpful response"},{"name":"Bad","description":"unhelpful or incorrect"}]'\
+"""
+
+_JUDGE_SPAN_TRIGGERS_HELP = """\
+JSON array of span filters that restrict which spans the judge evaluates. Omit to evaluate all spans.
+
+\b
+Shape: [{"span_name":"<name>","attribute_filters":[{"key":"<attr>","op":"=","value":"<string>"}]}]
+
+Example: --span-triggers '[{"span_name":"agent.run","attribute_filters":[]}]'\
+"""
+
+
 COMMAND_HELP: dict[str, str] = {
     "traces.search": (
         "Search traces in a project.\n"
@@ -103,6 +190,118 @@
         '  --sort-by \'{"field":"latency","direction":"desc"}\' \\\n'
         '  --pagination \'{"limit":25,"cursorSortValue":null,"cursorItemId":null}\''
     ),
+    "automations.create": (
+        "Create an automation (rule) in a project.\n"
+        "\n"
+        "An automation watches behavior/latency/cost metrics on the project and fires actions when its conditions match. Requires the developer role.\n"
+        "\n"
+        "\b\n"
+        "Example: judgment automations create <PROJECT_ID> my-rule any \\\n"
+        "  --description 'Alert on low relevance scores' \\\n"
+        "  --conditions '[{\"metric\":{\"type\":\"behavior\",\"behavior_id\":\"<BEHAVIOR_ID>\"},\"operator\":\"<\",\"threshold\":0.7}]' \\\n"
+        "  --actions '{\"slack\":[{\"channel\":\"#alerts\",\"webhook_url\":\"https://hooks.slack.com/...\"}]}' \\\n"
+        "  --cooldown-period 5 --cooldown-period-unit minutes"
+    ),
+    "automations.update": (
+        "Update an existing automation. All fields other than the positional arguments are optional — only supplied flags are applied.\n"
+        "\n"
+        "Use --active true/false to enable or disable without changing other fields. Requires the developer role.\n"
+        "\n"
+        "\b\n"
+        "Example: judgment automations update <PROJECT_ID> <RULE_ID> --active false"
+    ),
+    "automations.delete": (
+        "Delete an automation. Requires the admin role.\n"
+        "\n"
+        "\b\n"
+        "Example: judgment automations delete <PROJECT_ID> <RULE_ID>"
+    ),
+    "automations.get": (
+        "Fetch the full definition of a single automation by ID.\n"
+        "\n"
+        "\b\n"
+        "Example: judgment automations get <PROJECT_ID> <RULE_ID>"
+    ),
+    "behaviors.create-binary": (
+        "Create a binary (Yes/No) behavior backed by a prompt judge.\n"
+        "\n"
+        'Scores each trace/span as "Yes" (true) or "No" (false) using the prompt you supply. Pass --judge-id to attach to an existing judge instead of creating a new prompt scorer. Requires the developer role.\n'
+        "\n"
+        "\b\n"
+        "Example: judgment behaviors create-binary <PROJECT_ID> 'Relevance' \\\n"
+        "  'Was the assistant response relevant to the user message?' \\\n"
+        "  --description 'Binary relevance check' --model gpt-5.2"
+    ),
+    "behaviors.create-classifier": (
+        "Create a categorical (multi-class) behavior backed by a prompt judge.\n"
+        "\n"
+        "Scores each trace/span into one of the --options categories. Pass --judge-id to attach to an existing classifier judge instead of creating a new one. Requires the developer role.\n"
+        "\n"
+        "\b\n"
+        "Example: judgment behaviors create-classifier <PROJECT_ID> 'Sentiment' \\\n"
+        "  'Classify the assistant response tone.' \\\n"
+        "  --options '[{\"name\":\"Positive\"},{\"name\":\"Neutral\"},{\"name\":\"Negative\"}]'"
+    ),
+    "behaviors.update": (
+        "Update a behavior's description. Requires the developer role.\n"
+        "\n"
+        "\b\n"
+        "Example: judgment behaviors update <PROJECT_ID> <BEHAVIOR_ID> --description 'New description'"
+    ),
+    "behaviors.delete": (
+        "Delete a behavior. Requires the admin role.\n"
+        "\n"
+        "Binary behaviors always delete both the true and false rows. For classifier behaviors, pass --delete-all-values true to delete every category row for the judge at once. Pass --delete-scorer true to also delete the underlying prompt scorer when no other behaviors reference it.\n"
+        "\n"
+        "\b\n"
+        "Example: judgment behaviors delete <PROJECT_ID> <BEHAVIOR_ID> --delete-scorer true"
+    ),
+    "judges.update-settings": (
+        "Update a judge's online-evaluation configuration.\n"
+        "\n"
+        "Applies to all behaviors backed by the judge. evaluation_mode selects whether the judge runs continuously or on demand; sampling_rate is a percentage (0-100) of qualifying spans to score. Requires the developer role.\n"
+        "\n"
+        "\b\n"
+        "Example: judgment judges update-settings <PROJECT_ID> <JUDGE_ID> sampled 10 \\\n"
+        "  --span-triggers '[{\"span_name\":\"agent.run\",\"attribute_filters\":[]}]'"
+    ),
+    "projects.create": (
+        "Create a new project in your organization. Requires the developer role.\n"
+        "\n"
+        "\b\n"
+        "Example: judgment projects create my-new-project"
+    ),
+    "projects.add-favorite": (
+        "Mark a project as a favorite for your user so it appears pinned in the UI.\n"
+        "\n"
+        "\b\n"
+        "Example: judgment projects add-favorite <PROJECT_ID>"
+    ),
+    "projects.remove-favorite": (
+        "Remove a project from your user's favorites.\n"
+        "\n"
+        "\b\n"
+        "Example: judgment projects remove-favorite <PROJECT_ID>"
+    ),
+    "traces.add-tags": (
+        "Attach one or more string tags to an existing trace.\n"
+        "\n"
+        "Tags are additive — existing tags on the trace are preserved. Repeat --tags to add multiple tags.\n"
+        "\n"
+        "\b\n"
+        "Example: judgment traces add-tags <PROJECT_ID> <TRACE_ID> \\\n"
+        "  --tags regression --tags investigate"
+    ),
+    "traces.evaluate": (
+        "Queue traces for re-evaluation by the project's judges.\n"
+        "\n"
+        "Pass specific trace IDs with --trace-ids (repeat the flag) or use --evaluate-all true to re-evaluate every trace in the project. Use --specific-judge-names to restrict which judges run (repeat the flag). Requires the developer role.\n"
+        "\n"
+        "\b\n"
+        "Example: judgment traces evaluate <PROJECT_ID> \\\n"
+        "  --trace-ids <TRACE_ID_1> --trace-ids <TRACE_ID_2> \\\n"
+        "  --specific-judge-names Relevance"
+    ),
 }
 
 
@@ -119,6 +318,58 @@
         "time_range": _TIME_RANGE_HELP,
         "pagination": _SESSION_PAGINATION_HELP,
     },
+    "automations.create": {
+        "description": "Human-readable description shown in the UI.",
+        "conditions": _AUTOMATION_CONDITIONS_HELP,
+        "actions": _AUTOMATION_ACTIONS_HELP,
+        "cooldown_period": "Cooldown duration as a plain number. Paired with --cooldown-period-unit. Example: --cooldown-period 15 --cooldown-period-unit minutes.",
+        "cooldown_period_unit": "Unit for --cooldown-period: seconds | minutes | hours | days.",
+        "trigger_frequency_count": "Maximum triggers allowed within the window defined by --trigger-frequency-period and --trigger-frequency-period-unit.",
+        "trigger_frequency_period": "Window length for the trigger-frequency limit. Paired with --trigger-frequency-period-unit.",
+        "trigger_frequency_period_unit": "Unit for --trigger-frequency-period: seconds | minutes | hours | days.",
+    },
+    "automations.update": {
+        "name": "New name for the automation.",
+        "description": "New description for the automation.",
+        "conditions": _AUTOMATION_CONDITIONS_HELP,
+        "actions": _AUTOMATION_ACTIONS_HELP,
+        "active": "Enable (true) or disable (false) the automation without modifying other fields.",
+        "cooldown_period": _AUTOMATION_COOLDOWN_HELP,
+        "trigger_frequency": _AUTOMATION_TRIGGER_FREQUENCY_HELP,
+    },
+    "behaviors.create-binary": {
+        "description": "Human-readable description shown in the UI.",
+        "model": 'LLM model ID used by the judge prompt. Defaults to "gpt-5.2" when omitted.',
+        "category_ids": _BEHAVIOR_CATEGORY_IDS_HELP,
+        "advanced_settings": _BEHAVIOR_ADVANCED_SETTINGS_HELP,
+        "judge_id": "Attach the new behavior to an existing judge instead of creating one. The judge must be score_type=binary and have no existing behaviors.",
+    },
+    "behaviors.create-classifier": {
+        "options": _CLASSIFIER_OPTIONS_HELP,
+        "model": 'LLM model ID used by the judge prompt. Defaults to "gpt-5.2" when omitted.',
+        "category_ids": _BEHAVIOR_CATEGORY_IDS_HELP,
+        "advanced_settings": _BEHAVIOR_ADVANCED_SETTINGS_HELP,
+        "judge_id": "Attach the new behavior to an existing judge instead of creating one. The judge must be score_type=categorical and have no existing behaviors.",
+    },
+    "behaviors.update": {
+        "description": "New human-readable description for the behavior. Pass null to clear it.",
+    },
+    "behaviors.delete": {
+        "delete_scorer": "When true, also delete the underlying prompt scorer if no other behaviors reference it.",
+        "delete_all_values": "For classifier behaviors, when true deletes every category row for this judge (not just the provided BEHAVIOR_ID). Ignored for binary behaviors (those always delete both rows).",
+    },
+    "judges.update-settings": {
+        "span_triggers": _JUDGE_SPAN_TRIGGERS_HELP,
+        "session_scoring": "When true, run the judge at session granularity instead of per-span.",
+    },
+    "traces.add-tags": {
+        "tags": "Tag string to attach to the trace. Repeat --tags to attach multiple tags: --tags a --tags b.",
+    },
+    "traces.evaluate": {
+        "evaluate_all": "When true, re-evaluate every trace in the project. Mutually exclusive with --trace-ids.",
+        "trace_ids": "Trace UUID to re-evaluate. Repeat --trace-ids to evaluate multiple: --trace-ids <id1> --trace-ids <id2>.",
+        "specific_judge_names": "Restrict evaluation to judges with these names. Repeat the flag to pass multiple. Omit to run every applicable judge.",
+    },
 }
 
 
diff --git a/src/judgment_cli/generated_commands.py b/src/judgment_cli/generated_commands.py
index e7d8023..1e37354 100644
--- a/src/judgment_cli/generated_commands.py
+++ b/src/judgment_cli/generated_commands.py
@@ -21,6 +21,73 @@ def automations_group() -> None:
     pass
 
 
+@automations_group.command("create")
+@click.argument("project_id")
+@click.argument("name")
+@click.option("--description", "description", default=None, help='Human-readable description shown in the UI.')
+@click.option("--conditions", "conditions", required=True, help='JSON array of rule conditions. Each condition compares a behavior score or metric to a threshold. Items are ANDed or ORed together based on --combine-type ("all" vs "any").\n\n\x08\nCondition shape:\n  {"metric": {"type":<metric_type>, ...}, "operator": ">=|<=|>|<|==|!=", "threshold": <number>}\n\n\x08\nSupported metric types:\n  behavior       {"type":"behavior","behavior_id":"<uuid>"}\n  latency        {"type":"latency"}           (milliseconds)\n  cost           {"type":"cost"}              (USD)\n  token_count    {"type":"token_count"}\n\nExample: --conditions \'[{"metric":{"type":"behavior","behavior_id":"<uuid>"},"operator":">=","threshold":0.8}]\'')
+@click.argument("combine_type", type=click.Choice(['all', 'any']))
+@click.option("--actions", "actions", default=None, help='JSON object describing notification actions to fire when the automation triggers. Omit to store the automation with no actions.\n\n\x08\nShape: {"slack":[{"channel":"<channel>","webhook_url":"<url>"}], "pagerduty":[{"integration_key":"<key>"}], "email":[{"to":"<addr>"}]}\n\nExample: --actions \'{"slack":[{"channel":"#alerts","webhook_url":"https://hooks.slack.com/..."}]}\'')
+@click.option("--cooldown-period", "cooldown_period", default=None, type=float, help='Cooldown duration as a plain number. Paired with --cooldown-period-unit. Example: --cooldown-period 15 --cooldown-period-unit minutes.')
+@click.option("--cooldown-period-unit", "cooldown_period_unit", default=None, type=click.Choice(['seconds', 'minutes', 'hours', 'days']), help='Unit for --cooldown-period: seconds | minutes | hours | days.')
+@click.option("--trigger-frequency-count", "trigger_frequency_count", default=None, type=float, help='Maximum triggers allowed within the window defined by --trigger-frequency-period and --trigger-frequency-period-unit.')
+@click.option("--trigger-frequency-period", "trigger_frequency_period", default=None, type=float, help='Window length for the trigger-frequency limit. Paired with --trigger-frequency-period-unit.')
+@click.option("--trigger-frequency-period-unit", "trigger_frequency_period_unit", default=None, type=click.Choice(['seconds', 'minutes', 'hours', 'days']), help='Unit for --trigger-frequency-period: seconds | minutes | hours | days.')
+@click.pass_context
+def automations_create(ctx, project_id, name, description, conditions, combine_type, actions, cooldown_period, cooldown_period_unit, trigger_frequency_count, trigger_frequency_period, trigger_frequency_period_unit):
+    'Create an automation (rule) in a project.\n\n\x08\nAn automation watches behavior/latency/cost metrics on the project and fires actions when its conditions match. Requires the developer role.\n\n\x08\n\x08\nExample: judgment automations create <PROJECT_ID> my-rule any \\\n  --description \'Alert on low relevance scores\' \\\n  --conditions \'[{"metric":{"type":"behavior","behavior_id":"<BEHAVIOR_ID>"},"operator":"<","threshold":0.7}]\' \\\n  --actions \'{"slack":[{"channel":"#alerts","webhook_url":"https://hooks.slack.com/..."}]}\' \\\n  --cooldown-period 5 --cooldown-period-unit minutes'
+    url = "/automations/create"
+    body = {}
+    body["project_id"] = project_id
+    body["name"] = name
+    if description is not None:
+        body["description"] = description
+    body["conditions"] = json.loads(conditions)
+    body["combine_type"] = combine_type
+    if actions is not None:
+        body["actions"] = json.loads(actions)
+    if cooldown_period is not None:
+        body["cooldown_period"] = cooldown_period
+    if cooldown_period_unit is not None:
+        body["cooldown_period_unit"] = cooldown_period_unit
+    if trigger_frequency_count is not None:
+        body["trigger_frequency_count"] = trigger_frequency_count
+    if trigger_frequency_period is not None:
+        body["trigger_frequency_period"] = trigger_frequency_period
+    if trigger_frequency_period_unit is not None:
+        body["trigger_frequency_period_unit"] = trigger_frequency_period_unit
+    result = ctx.obj["client"].request("POST", url, json_body=body)
+    _output(result)
+
+
+@automations_group.command("delete")
+@click.argument("project_id")
+@click.argument("rule_id")
+@click.pass_context
+def automations_delete(ctx, project_id, rule_id):
+    'Delete an automation. Requires the admin role.\n\n\x08\n\x08\nExample: judgment automations delete <PROJECT_ID> <RULE_ID>'
+    url = "/automations/delete"
+    body = {}
+    body["project_id"] = project_id
+    body["rule_id"] = rule_id
+    result = ctx.obj["client"].request("POST", url, json_body=body)
+    _output(result)
+
+
+@automations_group.command("get")
+@click.argument("project_id")
+@click.argument("rule_id")
+@click.pass_context
+def automations_get(ctx, project_id, rule_id):
+    'Fetch the full definition of a single automation by ID.\n\n\x08\n\x08\nExample: judgment automations get <PROJECT_ID> <RULE_ID>'
+    url = "/automations/detail"
+    body = {}
+    body["project_id"] = project_id
+    body["rule_id"] = rule_id
+    result = ctx.obj["client"].request("POST", url, json_body=body)
+    _output(result)
+
+
 @automations_group.command("list")
 @click.argument("project_id")
 @click.pass_context
@@ -32,6 +99,44 @@ def automations_list(ctx, project_id):
     result = ctx.obj["client"].request("POST", url, json_body=body)
     _output(result)
 
+
+@automations_group.command("update")
+@click.argument("project_id")
+@click.argument("rule_id")
+@click.option("--name", "name", default=None, help='New name for the automation.')
+@click.option("--description", "description", default=None, help='New description for the automation.')
+@click.option("--conditions", "conditions", default=None, help='JSON array of rule conditions. Each condition compares a behavior score or metric to a threshold. Items are ANDed or ORed together based on --combine-type ("all" vs "any").\n\n\x08\nCondition shape:\n  {"metric": {"type":<metric_type>, ...}, "operator": ">=|<=|>|<|==|!=", "threshold": <number>}\n\n\x08\nSupported metric types:\n  behavior       {"type":"behavior","behavior_id":"<uuid>"}\n  latency        {"type":"latency"}           (milliseconds)\n  cost           {"type":"cost"}              (USD)\n  token_count    {"type":"token_count"}\n\nExample: --conditions \'[{"metric":{"type":"behavior","behavior_id":"<uuid>"},"operator":">=","threshold":0.8}]\'')
+@click.option("--combine-type", "combine_type", default=None, type=click.Choice(['all', 'any']))
+@click.option("--actions", "actions", default=None, help='JSON object describing notification actions to fire when the automation triggers. Omit to store the automation with no actions.\n\n\x08\nShape: {"slack":[{"channel":"<channel>","webhook_url":"<url>"}], "pagerduty":[{"integration_key":"<key>"}], "email":[{"to":"<addr>"}]}\n\nExample: --actions \'{"slack":[{"channel":"#alerts","webhook_url":"https://hooks.slack.com/..."}]}\'')
+@click.option("--active", "active", default=None, type=bool, help='Enable (true) or disable (false) the automation without modifying other fields.')
+@click.option("--cooldown-period", "cooldown_period", default=None, help='JSON 2-tuple [period, unit] describing the minimum wait between triggers.\n\n\x08\nShape: [<period:int>, <unit:"seconds"|"minutes"|"hours"|"days">]\n\nExample: --cooldown-period \'[15, "minutes"]\'  (at least 15 min between triggers)')
+@click.option("--trigger-frequency", "trigger_frequency", default=None, help='JSON 3-tuple [count, period, unit] describing the rate-limit window. Omit to disable frequency limiting.\n\n\x08\nShape: [<max_trigger_count:int>, <period:int>, <unit:"seconds"|"minutes"|"hours"|"days">]\n\nExample: --trigger-frequency \'[5, 1, "hours"]\'  (max 5 triggers per 1 hour)')
+@click.pass_context
+def automations_update(ctx, project_id, rule_id, name, description, conditions, combine_type, actions, active, cooldown_period, trigger_frequency):
+    'Update an existing automation. All fields other than the positional arguments are optional — only supplied flags are applied.\n\n\x08\nUse --active true/false to enable or disable without changing other fields. Requires the developer role.\n\n\x08\n\x08\nExample: judgment automations update <PROJECT_ID> <RULE_ID> --active false'
+    url = "/automations/update"
+    body = {}
+    body["project_id"] = project_id
+    body["rule_id"] = rule_id
+    if name is not None:
+        body["name"] = name
+    if description is not None:
+        body["description"] = description
+    if conditions is not None:
+        body["conditions"] = json.loads(conditions)
+    if combine_type is not None:
+        body["combine_type"] = combine_type
+    if actions is not None:
+        body["actions"] = json.loads(actions)
+    if active is not None:
+        body["active"] = active
+    if cooldown_period is not None:
+        body["cooldown_period"] = json.loads(cooldown_period)
+    if trigger_frequency is not None:
+        body["trigger_frequency"] = json.loads(trigger_frequency)
+    result = ctx.obj["client"].request("POST", url, json_body=body)
+    _output(result)
+
 # ────────────────────────────────────────────────────────────────────
 # Group: behaviors
 # ────────────────────────────────────────────────────────────────────
@@ -43,6 +148,87 @@ def behaviors_group() -> None:
     pass
 
 
+@behaviors_group.command("create-binary")
+@click.argument("project_id")
+@click.argument("name")
+@click.argument("prompt")
+@click.option("--description", "description", default=None, help='Human-readable description shown in the UI.')
+@click.option("--model", "model", default=None, help='LLM model ID used by the judge prompt. Defaults to "gpt-5.2" when omitted.')
+@click.option("--category-ids", "category_ids", multiple=True, help='UUID of a category to attach the behavior to. Repeat the flag to attach multiple categories: --category-ids <uuid1> --category-ids <uuid2>.')
+@click.option("--advanced-settings", "advanced_settings", default=None, help='JSON object overriding the judge\'s online evaluation configuration. If omitted, defaults are applied.\n\n\x08\nShape:\n  {\n    "online_evaluation_mode": "always" | "sampled" | "off",\n    "online_sampling_rate": <number 0-1>,\n    "online_span_triggers": [{"span_name":"<name>","attribute_filters":[...]}]?,\n    "online_session_scoring": <bool>?\n  }\n\nExample: --advanced-settings \'{"online_evaluation_mode":"sampled","online_sampling_rate":0.1}\'')
+@click.option("--judge-id", "judge_id", default=None, help='Attach the new behavior to an existing judge instead of creating one. The judge must be score_type=binary and have no existing behaviors.')
+@click.pass_context
+def behaviors_create_binary(ctx, project_id, name, prompt, description, model, category_ids, advanced_settings, judge_id):
+    'Create a binary (Yes/No) behavior backed by a prompt judge.\n\n\x08\nScores each trace/span as "Yes" (true) or "No" (false) using the prompt you supply. Pass --judge-id to attach to an existing judge instead of creating a new prompt scorer. Requires the developer role.\n\n\x08\n\x08\nExample: judgment behaviors create-binary <PROJECT_ID> \'Relevance\' \\\n  \'Was the assistant response relevant to the user message?\' \\\n  --description \'Binary relevance check\' --model gpt-5.2'
+    url = "/behaviors/create-binary"
+    body = {}
+    body["project_id"] = project_id
+    body["name"] = name
+    body["prompt"] = prompt
+    if description is not None:
+        body["description"] = description
+    if model is not None:
+        body["model"] = model
+    if category_ids:
+        body["category_ids"] = list(category_ids)
+    if advanced_settings is not None:
+        body["advanced_settings"] = json.loads(advanced_settings)
+    if judge_id is not None:
+        body["judge_id"] = judge_id
+    result = ctx.obj["client"].request("POST", url, json_body=body)
+    _output(result)
+
+
+@behaviors_group.command("create-classifier")
+@click.argument("project_id")
+@click.argument("name")
+@click.argument("prompt")
+@click.option("--options", "options", required=True, help='JSON array of the allowed output categories the classifier judge can return. Must contain at least one option.\n\n\x08\nShape:\n  [\n    {"name":"<label>", "description":"<optional human description>", "category_ids":["<uuid>", ...]},\n    ...\n  ]\n\nExample: --options \'[{"name":"Good","description":"helpful response"},{"name":"Bad","description":"unhelpful or incorrect"}]\'')
+@click.option("--model", "model", default=None, help='LLM model ID used by the judge prompt. Defaults to "gpt-5.2" when omitted.')
+@click.option("--category-ids", "category_ids", multiple=True, help='UUID of a category to attach the behavior to. Repeat the flag to attach multiple categories: --category-ids <uuid1> --category-ids <uuid2>.')
+@click.option("--advanced-settings", "advanced_settings", default=None, help='JSON object overriding the judge\'s online evaluation configuration. If omitted, defaults are applied.\n\n\x08\nShape:\n  {\n    "online_evaluation_mode": "always" | "sampled" | "off",\n    "online_sampling_rate": <number 0-1>,\n    "online_span_triggers": [{"span_name":"<name>","attribute_filters":[...]}]?,\n    "online_session_scoring": <bool>?\n  }\n\nExample: --advanced-settings \'{"online_evaluation_mode":"sampled","online_sampling_rate":0.1}\'')
+@click.option("--judge-id", "judge_id", default=None, help='Attach the new behavior to an existing judge instead of creating one. The judge must be score_type=categorical and have no existing behaviors.')
+@click.pass_context
+def behaviors_create_classifier(ctx, project_id, name, prompt, options, model, category_ids, advanced_settings, judge_id):
+    'Create a categorical (multi-class) behavior backed by a prompt judge.\n\n\x08\nScores each trace/span into one of the --options categories. Pass --judge-id to attach to an existing classifier judge instead of creating a new one. Requires the developer role.\n\n\x08\n\x08\nExample: judgment behaviors create-classifier <PROJECT_ID> \'Sentiment\' \\\n  \'Classify the assistant response tone.\' \\\n  --options \'[{"name":"Positive"},{"name":"Neutral"},{"name":"Negative"}]\''
+    url = "/behaviors/create-classifier"
+    body = {}
+    body["project_id"] = project_id
+    body["name"] = name
+    body["prompt"] = prompt
+    body["options"] = json.loads(options)
+    if model is not None:
+        body["model"] = model
+    if category_ids:
+        body["category_ids"] = list(category_ids)
+    if advanced_settings is not None:
+        body["advanced_settings"] = json.loads(advanced_settings)
+    if judge_id is not None:
+        body["judge_id"] = judge_id
+    result = ctx.obj["client"].request("POST", url, json_body=body)
+    _output(result)
+
+
+@behaviors_group.command("delete")
+@click.argument("project_id")
+@click.argument("behavior_id")
+@click.option("--delete-scorer", "delete_scorer", default=None, type=bool, help='When true, also delete the underlying prompt scorer if no other behaviors reference it.')
+@click.option("--delete-all-values", "delete_all_values", default=None, type=bool, help='For classifier behaviors, when true deletes every category row for this judge (not just the provided BEHAVIOR_ID). Ignored for binary behaviors (those always delete both rows).')
+@click.pass_context
+def behaviors_delete(ctx, project_id, behavior_id, delete_scorer, delete_all_values):
+    'Delete a behavior. Requires the admin role.\n\n\x08\nBinary behaviors always delete both the true and false rows. For classifier behaviors, pass --delete-all-values true to delete every category row for the judge at once. Pass --delete-scorer true to also delete the underlying prompt scorer when no other behaviors reference it.\n\n\x08\n\x08\nExample: judgment behaviors delete <PROJECT_ID> <BEHAVIOR_ID> --delete-scorer true'
+    url = "/behaviors/delete"
+    body = {}
+    body["project_id"] = project_id
+    body["behavior_id"] = behavior_id
+    if delete_scorer is not None:
+        body["delete_scorer"] = delete_scorer
+    if delete_all_values is not None:
+        body["delete_all_values"] = delete_all_values
+    result = ctx.obj["client"].request("POST", url, json_body=body)
+    _output(result)
+
+
 @behaviors_group.command("get")
 @click.argument("project_id")
 @click.argument("behavior_id")
@@ -74,6 +260,23 @@ def behaviors_list(ctx, project_id):
     result = ctx.obj["client"].request("POST", url, json_body=body)
     _output(result)
 
+
+@behaviors_group.command("update")
+@click.argument("project_id")
+@click.argument("behavior_id")
+@click.option("--description", "description", default=None, help='New human-readable description for the behavior. Pass null to clear it.')
+@click.pass_context
+def behaviors_update(ctx, project_id, behavior_id, description):
+    "Update a behavior's description. Requires the developer role.\n\n\x08\n\x08\nExample: judgment behaviors update <PROJECT_ID> <BEHAVIOR_ID> --description 'New description'"
+    url = "/behaviors/update"
+    body = {}
+    body["project_id"] = project_id
+    body["behavior_id"] = behavior_id
+    if description is not None:
+        body["description"] = description
+    result = ctx.obj["client"].request("POST", url, json_body=body)
+    _output(result)
+
 # ────────────────────────────────────────────────────────────────────
 # Group: docs
 # ────────────────────────────────────────────────────────────────────
@@ -135,6 +338,30 @@ def judges_settings(ctx, project_id, judge_id):
     result = ctx.obj["client"].request("POST", url, json_body=body)
     _output(result)
 
+
+@judges_group.command("update-settings")
+@click.argument("project_id")
+@click.argument("judge_id")
+@click.argument("evaluation_mode", type=click.Choice(['continuous', 'on_demand']))
+@click.argument("sampling_rate", type=float)
+@click.option("--span-triggers", "span_triggers", default=None, help='JSON array of span filters that restrict which spans the judge evaluates. Omit to evaluate all spans.\n\n\x08\nShape: [{"span_name":"<name>","attribute_filters":[{"key":"<attr>","op":"=","value":"<string>"}]}]\n\nExample: --span-triggers \'[{"span_name":"agent.run","attribute_filters":[]}]\'')
+@click.option("--session-scoring", "session_scoring", default=None, type=bool, help='When true, run the judge at session granularity instead of per-span.')
+@click.pass_context
+def judges_update_settings(ctx, project_id, judge_id, evaluation_mode, sampling_rate, span_triggers, session_scoring):
+    'Update a judge\'s online-evaluation configuration.\n\n\x08\nApplies to all behaviors backed by the judge. evaluation_mode selects whether the judge runs continuously or on demand; sampling_rate is a percentage (0-100) of qualifying spans to score. Requires the developer role.\n\n\x08\n\x08\nExample: judgment judges update-settings <PROJECT_ID> <JUDGE_ID> sampled 10 \\\n  --span-triggers \'[{"span_name":"agent.run","attribute_filters":[]}]\''
+    url = "/judges/update-settings"
+    body = {}
+    body["project_id"] = project_id
+    body["judge_id"] = judge_id
+    body["evaluation_mode"] = evaluation_mode
+    body["sampling_rate"] = sampling_rate
+    if span_triggers is not None:
+        body["span_triggers"] = json.loads(span_triggers)
+    if session_scoring is not None:
+        body["session_scoring"] = session_scoring
+    result = ctx.obj["client"].request("POST", url, json_body=body)
+    _output(result)
+
 # ────────────────────────────────────────────────────────────────────
 # Group: projects
 # ────────────────────────────────────────────────────────────────────
@@ -146,6 +373,30 @@ def projects_group() -> None:
     pass
 
 
+@projects_group.command("add-favorite")
+@click.argument("project_id")
+@click.pass_context
+def projects_add_favorite(ctx, project_id):
+    'Mark a project as a favorite for your user so it appears pinned in the UI.\n\n\x08\n\x08\nExample: judgment projects add-favorite <PROJECT_ID>'
+    url = "/projects/add-favorite"
+    body = {}
+    body["project_id"] = project_id
+    result = ctx.obj["client"].request("POST", url, json_body=body)
+    _output(result)
+
+
+@projects_group.command("create")
+@click.argument("project_name")
+@click.pass_context
+def projects_create(ctx, project_name):
+    'Create a new project in your organization. Requires the developer role.\n\n\x08\n\x08\nExample: judgment projects create my-new-project'
+    url = "/projects/create"
+    body = {}
+    body["project_name"] = project_name
+    result = ctx.obj["client"].request("POST", url, json_body=body)
+    _output(result)
+
+
 @projects_group.command("list")
 @click.pass_context
 def projects_list(ctx):
@@ -154,6 +405,18 @@ def projects_list(ctx):
     result = ctx.obj["client"].request("GET", url)
     _output(result)
 
+
+@projects_group.command("remove-favorite")
+@click.argument("project_id")
+@click.pass_context
+def projects_remove_favorite(ctx, project_id):
+    "Remove a project from your user's favorites.\n\n\x08\n\x08\nExample: judgment projects remove-favorite <PROJECT_ID>"
+    url = "/projects/remove-favorite"
+    body = {}
+    body["project_id"] = project_id
+    result = ctx.obj["client"].request("POST", url, json_body=body)
+    _output(result)
+
 # ────────────────────────────────────────────────────────────────────
 # Group: sessions
 # ────────────────────────────────────────────────────────────────────
@@ -240,6 +503,23 @@ def traces_group() -> None:
     pass
 
 
+@traces_group.command("add-tags")
+@click.argument("project_id")
+@click.argument("trace_id")
+@click.option("--tags", "tags", multiple=True, help='Tag string to attach to the trace. Repeat --tags to attach multiple tags: --tags a --tags b.')
+@click.pass_context
+def traces_add_tags(ctx, project_id, trace_id, tags):
+    'Attach one or more string tags to an existing trace.\n\n\x08\nTags are additive — existing tags on the trace are preserved. Repeat --tags to add multiple tags.\n\n\x08\n\x08\nExample: judgment traces add-tags <PROJECT_ID> <TRACE_ID> \\\n  --tags regression --tags investigate'
+    url = "/traces/add-tags"
+    body = {}
+    body["project_id"] = project_id
+    body["trace_id"] = trace_id
+    if tags:
+        body["tags"] = list(tags)
+    result = ctx.obj["client"].request("POST", url, json_body=body)
+    _output(result)
+
+
 @traces_group.command("behaviors")
 @click.argument("project_id")
 @click.argument("trace_id")
@@ -254,6 +534,27 @@ def traces_behaviors(ctx, project_id, trace_id):
     _output(result)
 
 
+@traces_group.command("evaluate")
+@click.argument("project_id")
+@click.option("--evaluate-all", "evaluate_all", default=None, type=bool, help='When true, re-evaluate every trace in the project. Mutually exclusive with --trace-ids.')
+@click.option("--trace-ids", "trace_ids", multiple=True, help='Trace UUID to re-evaluate. Repeat --trace-ids to evaluate multiple: --trace-ids <id1> --trace-ids <id2>.')
+@click.option("--specific-judge-names", "specific_judge_names", multiple=True, help='Restrict evaluation to judges with these names. Repeat the flag to pass multiple. Omit to run every applicable judge.')
+@click.pass_context
+def traces_evaluate(ctx, project_id, evaluate_all, trace_ids, specific_judge_names):
+    "Queue traces for re-evaluation by the project's judges.\n\n\x08\nPass specific trace IDs with --trace-ids (repeat the flag) or use --evaluate-all true to re-evaluate every trace in the project. Use --specific-judge-names to restrict which judges run (repeat the flag). Requires the developer role.\n\n\x08\n\x08\nExample: judgment traces evaluate <PROJECT_ID> \\\n  --trace-ids <TRACE_ID_1> --trace-ids <TRACE_ID_2> \\\n  --specific-judge-names Relevance"
+    url = "/traces/evaluate"
+    body = {}
+    body["project_id"] = project_id
+    if evaluate_all is not None:
+        body["evaluate_all"] = evaluate_all
+    if trace_ids:
+        body["trace_ids"] = list(trace_ids)
+    if specific_judge_names:
+        body["specific_judge_names"] = list(specific_judge_names)
+    result = ctx.obj["client"].request("POST", url, json_body=body)
+    _output(result)
+
+
 @traces_group.command("get")
 @click.argument("project_id")
 @click.argument("trace_id")

From bcaebc8c2e46c55bedb0186a3fcaeba7af893b49 Mon Sep 17 00:00:00 2001
From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Date: Fri, 24 Apr 2026 00:32:35 +0000
Subject: [PATCH 2/6] Correct help text to match server schemas (conditions,
 advanced-settings, span triggers, evaluation_mode)

Co-Authored-By: Aaryan Divate <divateaaryan@gmail.com>
---
 scripts/help_overrides.py              | 100 +++++++++++++++++--------
 src/judgment_cli/generated_commands.py |  22 +++---
 2 files changed, 81 insertions(+), 41 deletions(-)

diff --git a/scripts/help_overrides.py b/scripts/help_overrides.py
index 8b4b0fc..b7a76b1 100644
--- a/scripts/help_overrides.py
+++ b/scripts/help_overrides.py
@@ -81,62 +81,91 @@
 
 
 _AUTOMATION_CONDITIONS_HELP = """\
-JSON array of rule conditions. Each condition compares a behavior score or metric to a threshold. Items are ANDed or ORed together based on --combine-type ("all" vs "any").
+JSON array of rule conditions. Each condition references a named metric/scorer on the project and a comparison. Items are ANDed or ORed together based on the COMBINE_TYPE positional argument ("all" vs "any").
 
 \b
 Condition shape:
-  {"metric": {"type":<metric_type>, ...}, "operator": ">=|<=|>|<|==|!=", "threshold": <number>}
+  {
+    "metric": {
+      "scorer_type": "behavior" | "judge" | "prompt" | "custom" | "static" | "span_attribute" | "error",
+      "name": "<scorer or metric name>",
+      "threshold": <number | string | null>?
+    },
+    "comparison": "lt" | "gt" | "eq" | "gte" | "lte" | "fails" | "succeeds" | "chooses" | "detected" | "equals" | "contains" | "exists"
+  }
 
 \b
-Supported metric types:
-  behavior       {"type":"behavior","behavior_id":"<uuid>"}
-  latency        {"type":"latency"}           (milliseconds)
-  cost           {"type":"cost"}              (USD)
-  token_count    {"type":"token_count"}
-
-Example: --conditions '[{"metric":{"type":"behavior","behavior_id":"<uuid>"},"operator":">=","threshold":0.8}]'\
+Common scorer_type values:
+  behavior        Judge-scored behavior (name = behavior name, e.g. "Relevance")
+  static          Built-in metrics like "duration" (ms) or "llm_cost" (USD)
+  prompt/custom   Prompt or custom scorer by name
+  span_attribute  Arbitrary span attribute key (name = attribute key)
+  error           Span error condition
+
+Example: --conditions '[{"metric":{"scorer_type":"behavior","name":"Relevance","threshold":0.7},"comparison":"gte"}]'\
 """
 
 _AUTOMATION_ACTIONS_HELP = """\
-JSON object describing notification actions to fire when the automation triggers. Omit to store the automation with no actions.
+JSON object describing what happens when the automation fires. All top-level keys are optional — include only the actions you want configured.
 
 \b
-Shape: {"slack":[{"channel":"<channel>","webhook_url":"<url>"}], "pagerduty":[{"integration_key":"<key>"}], "email":[{"to":"<addr>"}]}
+Shape:
+  {
+    "notification": {
+      "enabled": <bool>?,
+      "communication_methods": ["email" | "slack" | "pagerduty"],
+      "email_addresses": ["<addr>", ...]?,
+      "pagerduty_config": {"routing_key":"<key>","severity":"critical"|"error"|"warning"|"info"}?
+    }?,
+    "dataset_addition": {
+      "enabled": <bool>?,
+      "dataset_name": "<dataset>",
+      "metadata_fields": <any>?
+    }?,
+    "behavior_evaluation": {
+      "enabled": <bool>?,
+      "behavior_judge_names": ["<judge_name>", ...]
+    }?
+  }
 
-Example: --actions '{"slack":[{"channel":"#alerts","webhook_url":"https://hooks.slack.com/..."}]}'\
+Slack notifications are configured per-organization in the Judgment UI; pass "slack" in communication_methods to use them. Example: --actions '{"notification":{"communication_methods":["slack"]}}'\
 """
 
 _AUTOMATION_TRIGGER_FREQUENCY_HELP = """\
-JSON 3-tuple [count, period, unit] describing the rate-limit window. Omit to disable frequency limiting.
+JSON 3-tuple [count, period, unit] describing the rate-limit window. Pass null to disable frequency limiting.
 
 \b
-Shape: [<max_trigger_count:int>, <period:int>, <unit:"seconds"|"minutes"|"hours"|"days">]
+Shape: [<max_trigger_count:number>, <period:number>, <unit:"seconds"|"minutes"|"hours"|"days">]
 
 Example: --trigger-frequency '[5, 1, "hours"]'  (max 5 triggers per 1 hour)\
 """
 
 _AUTOMATION_COOLDOWN_HELP = """\
-JSON 2-tuple [period, unit] describing the minimum wait between triggers.
+JSON 2-tuple [period, unit] describing the minimum wait between triggers. Pass null to clear the cooldown.
 
 \b
-Shape: [<period:int>, <unit:"seconds"|"minutes"|"hours"|"days">]
+Shape: [<period:number>, <unit:"seconds"|"minutes"|"hours"|"days">]
 
 Example: --cooldown-period '[15, "minutes"]'  (at least 15 min between triggers)\
 """
 
 _BEHAVIOR_ADVANCED_SETTINGS_HELP = """\
-JSON object overriding the judge's online evaluation configuration. If omitted, defaults are applied.
+JSON object overriding the judge's online-evaluation configuration. All four fields are required when this flag is supplied.
 
 \b
 Shape:
   {
-    "online_evaluation_mode": "always" | "sampled" | "off",
-    "online_sampling_rate": <number 0-1>,
-    "online_span_triggers": [{"span_name":"<name>","attribute_filters":[...]}]?,
-    "online_session_scoring": <bool>?
+    "online_evaluation_mode": "continuous" | "on_demand",
+    "online_sampling_rate": <number 0-100>,
+    "online_span_triggers": [
+      {"field":"span_name"|"span_attribute","operator":"contains"|"equals"|"exists","value":"<string>","key":"<attr-key>"?}
+    ],
+    "online_session_scoring": <bool>
   }
 
-Example: --advanced-settings '{"online_evaluation_mode":"sampled","online_sampling_rate":0.1}'\
+"continuous" runs the judge automatically on qualifying spans; "on_demand" requires a manual `judgment traces evaluate` call. online_sampling_rate is a percent of matching spans to score.
+
+Example: --advanced-settings '{"online_evaluation_mode":"continuous","online_sampling_rate":10,"online_span_triggers":[],"online_session_scoring":false}'\
 """
 
 _BEHAVIOR_CATEGORY_IDS_HELP = (
@@ -158,12 +187,23 @@
 """
 
 _JUDGE_SPAN_TRIGGERS_HELP = """\
-JSON array of span filters that restrict which spans the judge evaluates. Omit to evaluate all spans.
+JSON array of span filters that restrict which spans the judge evaluates. Pass [] to evaluate all spans.
 
 \b
-Shape: [{"span_name":"<name>","attribute_filters":[{"key":"<attr>","op":"=","value":"<string>"}]}]
+Shape:
+  [
+    {
+      "field": "span_name" | "span_attribute",
+      "operator": "contains" | "equals" | "exists",
+      "value": "<string>",
+      "key": "<attribute key>"?
+    },
+    ...
+  ]
+
+Use "field":"span_name" to match on span names; "field":"span_attribute" with "key":"<attr>" to match on a span attribute's value. Triggers are ANDed together — a span must match every entry to be evaluated.
 
-Example: --span-triggers '[{"span_name":"agent.run","attribute_filters":[]}]'\
+Example: --span-triggers '[{"field":"span_name","operator":"equals","value":"agent.run"}]'\
 """
 
 
@@ -198,8 +238,8 @@
         "\b\n"
         "Example: judgment automations create <PROJECT_ID> my-rule any \\\n"
         "  --description 'Alert on low relevance scores' \\\n"
-        "  --conditions '[{\"metric\":{\"type\":\"behavior\",\"behavior_id\":\"<BEHAVIOR_ID>\"},\"operator\":\"<\",\"threshold\":0.7}]' \\\n"
-        "  --actions '{\"slack\":[{\"channel\":\"#alerts\",\"webhook_url\":\"https://hooks.slack.com/...\"}]}' \\\n"
+        "  --conditions '[{\"metric\":{\"scorer_type\":\"behavior\",\"name\":\"Relevance\",\"threshold\":0.7},\"comparison\":\"lt\"}]' \\\n"
+        "  --actions '{\"notification\":{\"communication_methods\":[\"slack\"]}}' \\\n"
         "  --cooldown-period 5 --cooldown-period-unit minutes"
     ),
     "automations.update": (
@@ -259,11 +299,11 @@
     "judges.update-settings": (
         "Update a judge's online-evaluation configuration.\n"
         "\n"
-        "Applies to all behaviors backed by the judge. evaluation_mode selects whether the judge runs continuously or on demand; sampling_rate is a percentage (0-100) of qualifying spans to score. Requires the developer role.\n"
+        "Applies to all behaviors backed by the judge. EVALUATION_MODE selects whether the judge runs continuously on every matching span (\"continuous\") or only when explicitly invoked (\"on_demand\"); SAMPLING_RATE is a percentage (0-100) of qualifying spans to score. Requires the developer role.\n"
         "\n"
         "\b\n"
-        "Example: judgment judges update-settings <PROJECT_ID> <JUDGE_ID> sampled 10 \\\n"
-        "  --span-triggers '[{\"span_name\":\"agent.run\",\"attribute_filters\":[]}]'"
+        "Example: judgment judges update-settings <PROJECT_ID> <JUDGE_ID> continuous 10 \\\n"
+        "  --span-triggers '[{\"field\":\"span_name\",\"operator\":\"equals\",\"value\":\"agent.run\"}]'"
     ),
     "projects.create": (
         "Create a new project in your organization. Requires the developer role.\n"
diff --git a/src/judgment_cli/generated_commands.py b/src/judgment_cli/generated_commands.py
index 1e37354..1275f87 100644
--- a/src/judgment_cli/generated_commands.py
+++ b/src/judgment_cli/generated_commands.py
@@ -25,9 +25,9 @@ def automations_group() -> None:
 @click.argument("project_id")
 @click.argument("name")
 @click.option("--description", "description", default=None, help='Human-readable description shown in the UI.')
-@click.option("--conditions", "conditions", required=True, help='JSON array of rule conditions. Each condition compares a behavior score or metric to a threshold. Items are ANDed or ORed together based on --combine-type ("all" vs "any").\n\n\x08\nCondition shape:\n  {"metric": {"type":<metric_type>, ...}, "operator": ">=|<=|>|<|==|!=", "threshold": <number>}\n\n\x08\nSupported metric types:\n  behavior       {"type":"behavior","behavior_id":"<uuid>"}\n  latency        {"type":"latency"}           (milliseconds)\n  cost           {"type":"cost"}              (USD)\n  token_count    {"type":"token_count"}\n\nExample: --conditions \'[{"metric":{"type":"behavior","behavior_id":"<uuid>"},"operator":">=","threshold":0.8}]\'')
+@click.option("--conditions", "conditions", required=True, help='JSON array of rule conditions. Each condition references a named metric/scorer on the project and a comparison. Items are ANDed or ORed together based on the COMBINE_TYPE positional argument ("all" vs "any").\n\n\x08\nCondition shape:\n  {\n    "metric": {\n      "scorer_type": "behavior" | "judge" | "prompt" | "custom" | "static" | "span_attribute" | "error",\n      "name": "<scorer or metric name>",\n      "threshold": <number | string | null>?\n    },\n    "comparison": "lt" | "gt" | "eq" | "gte" | "lte" | "fails" | "succeeds" | "chooses" | "detected" | "equals" | "contains" | "exists"\n  }\n\n\x08\nCommon scorer_type values:\n  behavior        Judge-scored behavior (name = behavior name, e.g. "Relevance")\n  static          Built-in metrics like "duration" (ms) or "llm_cost" (USD)\n  prompt/custom   Prompt or custom scorer by name\n  span_attribute  Arbitrary span attribute key (name = attribute key)\n  error           Span error condition\n\nExample: --conditions \'[{"metric":{"scorer_type":"behavior","name":"Relevance","threshold":0.7},"comparison":"gte"}]\'')
 @click.argument("combine_type", type=click.Choice(['all', 'any']))
-@click.option("--actions", "actions", default=None, help='JSON object describing notification actions to fire when the automation triggers. Omit to store the automation with no actions.\n\n\x08\nShape: {"slack":[{"channel":"<channel>","webhook_url":"<url>"}], "pagerduty":[{"integration_key":"<key>"}], "email":[{"to":"<addr>"}]}\n\nExample: --actions \'{"slack":[{"channel":"#alerts","webhook_url":"https://hooks.slack.com/..."}]}\'')
+@click.option("--actions", "actions", default=None, help='JSON object describing what happens when the automation fires. All top-level keys are optional — include only the actions you want configured.\n\n\x08\nShape:\n  {\n    "notification": {\n      "enabled": <bool>?,\n      "communication_methods": ["email" | "slack" | "pagerduty"],\n      "email_addresses": ["<addr>", ...]?,\n      "pagerduty_config": {"routing_key":"<key>","severity":"critical"|"error"|"warning"|"info"}?\n    }?,\n    "dataset_addition": {\n      "enabled": <bool>?,\n      "dataset_name": "<dataset>",\n      "metadata_fields": <any>?\n    }?,\n    "behavior_evaluation": {\n      "enabled": <bool>?,\n      "behavior_judge_names": ["<judge_name>", ...]\n    }?\n  }\n\nSlack notifications are configured per-organization in the Judgment UI; pass "slack" in communication_methods to use them. Example: --actions \'{"notification":{"communication_methods":["slack"]}}\'')
 @click.option("--cooldown-period", "cooldown_period", default=None, type=float, help='Cooldown duration as a plain number. Paired with --cooldown-period-unit. Example: --cooldown-period 15 --cooldown-period-unit minutes.')
 @click.option("--cooldown-period-unit", "cooldown_period_unit", default=None, type=click.Choice(['seconds', 'minutes', 'hours', 'days']), help='Unit for --cooldown-period: seconds | minutes | hours | days.')
 @click.option("--trigger-frequency-count", "trigger_frequency_count", default=None, type=float, help='Maximum triggers allowed within the window defined by --trigger-frequency-period and --trigger-frequency-period-unit.')
@@ -35,7 +35,7 @@ def automations_group() -> None:
 @click.option("--trigger-frequency-period-unit", "trigger_frequency_period_unit", default=None, type=click.Choice(['seconds', 'minutes', 'hours', 'days']), help='Unit for --trigger-frequency-period: seconds | minutes | hours | days.')
 @click.pass_context
 def automations_create(ctx, project_id, name, description, conditions, combine_type, actions, cooldown_period, cooldown_period_unit, trigger_frequency_count, trigger_frequency_period, trigger_frequency_period_unit):
-    'Create an automation (rule) in a project.\n\n\x08\nAn automation watches behavior/latency/cost metrics on the project and fires actions when its conditions match. Requires the developer role.\n\n\x08\n\x08\nExample: judgment automations create <PROJECT_ID> my-rule any \\\n  --description \'Alert on low relevance scores\' \\\n  --conditions \'[{"metric":{"type":"behavior","behavior_id":"<BEHAVIOR_ID>"},"operator":"<","threshold":0.7}]\' \\\n  --actions \'{"slack":[{"channel":"#alerts","webhook_url":"https://hooks.slack.com/..."}]}\' \\\n  --cooldown-period 5 --cooldown-period-unit minutes'
+    'Create an automation (rule) in a project.\n\n\x08\nAn automation watches behavior/latency/cost metrics on the project and fires actions when its conditions match. Requires the developer role.\n\n\x08\n\x08\nExample: judgment automations create <PROJECT_ID> my-rule any \\\n  --description \'Alert on low relevance scores\' \\\n  --conditions \'[{"metric":{"scorer_type":"behavior","name":"Relevance","threshold":0.7},"comparison":"lt"}]\' \\\n  --actions \'{"notification":{"communication_methods":["slack"]}}\' \\\n  --cooldown-period 5 --cooldown-period-unit minutes'
     url = "/automations/create"
     body = {}
     body["project_id"] = project_id
@@ -105,12 +105,12 @@ def automations_list(ctx, project_id):
 @click.argument("rule_id")
 @click.option("--name", "name", default=None, help='New name for the automation.')
 @click.option("--description", "description", default=None, help='New description for the automation.')
-@click.option("--conditions", "conditions", default=None, help='JSON array of rule conditions. Each condition compares a behavior score or metric to a threshold. Items are ANDed or ORed together based on --combine-type ("all" vs "any").\n\n\x08\nCondition shape:\n  {"metric": {"type":<metric_type>, ...}, "operator": ">=|<=|>|<|==|!=", "threshold": <number>}\n\n\x08\nSupported metric types:\n  behavior       {"type":"behavior","behavior_id":"<uuid>"}\n  latency        {"type":"latency"}           (milliseconds)\n  cost           {"type":"cost"}              (USD)\n  token_count    {"type":"token_count"}\n\nExample: --conditions \'[{"metric":{"type":"behavior","behavior_id":"<uuid>"},"operator":">=","threshold":0.8}]\'')
+@click.option("--conditions", "conditions", default=None, help='JSON array of rule conditions. Each condition references a named metric/scorer on the project and a comparison. Items are ANDed or ORed together based on the COMBINE_TYPE positional argument ("all" vs "any").\n\n\x08\nCondition shape:\n  {\n    "metric": {\n      "scorer_type": "behavior" | "judge" | "prompt" | "custom" | "static" | "span_attribute" | "error",\n      "name": "<scorer or metric name>",\n      "threshold": <number | string | null>?\n    },\n    "comparison": "lt" | "gt" | "eq" | "gte" | "lte" | "fails" | "succeeds" | "chooses" | "detected" | "equals" | "contains" | "exists"\n  }\n\n\x08\nCommon scorer_type values:\n  behavior        Judge-scored behavior (name = behavior name, e.g. "Relevance")\n  static          Built-in metrics like "duration" (ms) or "llm_cost" (USD)\n  prompt/custom   Prompt or custom scorer by name\n  span_attribute  Arbitrary span attribute key (name = attribute key)\n  error           Span error condition\n\nExample: --conditions \'[{"metric":{"scorer_type":"behavior","name":"Relevance","threshold":0.7},"comparison":"gte"}]\'')
 @click.option("--combine-type", "combine_type", default=None, type=click.Choice(['all', 'any']))
-@click.option("--actions", "actions", default=None, help='JSON object describing notification actions to fire when the automation triggers. Omit to store the automation with no actions.\n\n\x08\nShape: {"slack":[{"channel":"<channel>","webhook_url":"<url>"}], "pagerduty":[{"integration_key":"<key>"}], "email":[{"to":"<addr>"}]}\n\nExample: --actions \'{"slack":[{"channel":"#alerts","webhook_url":"https://hooks.slack.com/..."}]}\'')
+@click.option("--actions", "actions", default=None, help='JSON object describing what happens when the automation fires. All top-level keys are optional — include only the actions you want configured.\n\n\x08\nShape:\n  {\n    "notification": {\n      "enabled": <bool>?,\n      "communication_methods": ["email" | "slack" | "pagerduty"],\n      "email_addresses": ["<addr>", ...]?,\n      "pagerduty_config": {"routing_key":"<key>","severity":"critical"|"error"|"warning"|"info"}?\n    }?,\n    "dataset_addition": {\n      "enabled": <bool>?,\n      "dataset_name": "<dataset>",\n      "metadata_fields": <any>?\n    }?,\n    "behavior_evaluation": {\n      "enabled": <bool>?,\n      "behavior_judge_names": ["<judge_name>", ...]\n    }?\n  }\n\nSlack notifications are configured per-organization in the Judgment UI; pass "slack" in communication_methods to use them. Example: --actions \'{"notification":{"communication_methods":["slack"]}}\'')
 @click.option("--active", "active", default=None, type=bool, help='Enable (true) or disable (false) the automation without modifying other fields.')
-@click.option("--cooldown-period", "cooldown_period", default=None, help='JSON 2-tuple [period, unit] describing the minimum wait between triggers.\n\n\x08\nShape: [<period:int>, <unit:"seconds"|"minutes"|"hours"|"days">]\n\nExample: --cooldown-period \'[15, "minutes"]\'  (at least 15 min between triggers)')
-@click.option("--trigger-frequency", "trigger_frequency", default=None, help='JSON 3-tuple [count, period, unit] describing the rate-limit window. Omit to disable frequency limiting.\n\n\x08\nShape: [<max_trigger_count:int>, <period:int>, <unit:"seconds"|"minutes"|"hours"|"days">]\n\nExample: --trigger-frequency \'[5, 1, "hours"]\'  (max 5 triggers per 1 hour)')
+@click.option("--cooldown-period", "cooldown_period", default=None, help='JSON 2-tuple [period, unit] describing the minimum wait between triggers. Pass null to clear the cooldown.\n\n\x08\nShape: [<period:number>, <unit:"seconds"|"minutes"|"hours"|"days">]\n\nExample: --cooldown-period \'[15, "minutes"]\'  (at least 15 min between triggers)')
+@click.option("--trigger-frequency", "trigger_frequency", default=None, help='JSON 3-tuple [count, period, unit] describing the rate-limit window. Pass null to disable frequency limiting.\n\n\x08\nShape: [<max_trigger_count:number>, <period:number>, <unit:"seconds"|"minutes"|"hours"|"days">]\n\nExample: --trigger-frequency \'[5, 1, "hours"]\'  (max 5 triggers per 1 hour)')
 @click.pass_context
 def automations_update(ctx, project_id, rule_id, name, description, conditions, combine_type, actions, active, cooldown_period, trigger_frequency):
     'Update an existing automation. All fields other than the positional arguments are optional — only supplied flags are applied.\n\n\x08\nUse --active true/false to enable or disable without changing other fields. Requires the developer role.\n\n\x08\n\x08\nExample: judgment automations update <PROJECT_ID> <RULE_ID> --active false'
@@ -155,7 +155,7 @@ def behaviors_group() -> None:
 @click.option("--description", "description", default=None, help='Human-readable description shown in the UI.')
 @click.option("--model", "model", default=None, help='LLM model ID used by the judge prompt. Defaults to "gpt-5.2" when omitted.')
 @click.option("--category-ids", "category_ids", multiple=True, help='UUID of a category to attach the behavior to. Repeat the flag to attach multiple categories: --category-ids <uuid1> --category-ids <uuid2>.')
-@click.option("--advanced-settings", "advanced_settings", default=None, help='JSON object overriding the judge\'s online evaluation configuration. If omitted, defaults are applied.\n\n\x08\nShape:\n  {\n    "online_evaluation_mode": "always" | "sampled" | "off",\n    "online_sampling_rate": <number 0-1>,\n    "online_span_triggers": [{"span_name":"<name>","attribute_filters":[...]}]?,\n    "online_session_scoring": <bool>?\n  }\n\nExample: --advanced-settings \'{"online_evaluation_mode":"sampled","online_sampling_rate":0.1}\'')
+@click.option("--advanced-settings", "advanced_settings", default=None, help='JSON object overriding the judge\'s online-evaluation configuration. All four fields are required when this flag is supplied.\n\n\x08\nShape:\n  {\n    "online_evaluation_mode": "continuous" | "on_demand",\n    "online_sampling_rate": <number 0-100>,\n    "online_span_triggers": [\n      {"field":"span_name"|"span_attribute","operator":"contains"|"equals"|"exists","value":"<string>","key":"<attr-key>"?}\n    ],\n    "online_session_scoring": <bool>\n  }\n\n"continuous" runs the judge automatically on qualifying spans; "on_demand" requires a manual `judgment traces evaluate` call. online_sampling_rate is a percent of matching spans to score.\n\nExample: --advanced-settings \'{"online_evaluation_mode":"continuous","online_sampling_rate":10,"online_span_triggers":[],"online_session_scoring":false}\'')
 @click.option("--judge-id", "judge_id", default=None, help='Attach the new behavior to an existing judge instead of creating one. The judge must be score_type=binary and have no existing behaviors.')
 @click.pass_context
 def behaviors_create_binary(ctx, project_id, name, prompt, description, model, category_ids, advanced_settings, judge_id):
@@ -186,7 +186,7 @@ def behaviors_create_binary(ctx, project_id, name, prompt, description, model, c
 @click.option("--options", "options", required=True, help='JSON array of the allowed output categories the classifier judge can return. Must contain at least one option.\n\n\x08\nShape:\n  [\n    {"name":"<label>", "description":"<optional human description>", "category_ids":["<uuid>", ...]},\n    ...\n  ]\n\nExample: --options \'[{"name":"Good","description":"helpful response"},{"name":"Bad","description":"unhelpful or incorrect"}]\'')
 @click.option("--model", "model", default=None, help='LLM model ID used by the judge prompt. Defaults to "gpt-5.2" when omitted.')
 @click.option("--category-ids", "category_ids", multiple=True, help='UUID of a category to attach the behavior to. Repeat the flag to attach multiple categories: --category-ids <uuid1> --category-ids <uuid2>.')
-@click.option("--advanced-settings", "advanced_settings", default=None, help='JSON object overriding the judge\'s online evaluation configuration. If omitted, defaults are applied.\n\n\x08\nShape:\n  {\n    "online_evaluation_mode": "always" | "sampled" | "off",\n    "online_sampling_rate": <number 0-1>,\n    "online_span_triggers": [{"span_name":"<name>","attribute_filters":[...]}]?,\n    "online_session_scoring": <bool>?\n  }\n\nExample: --advanced-settings \'{"online_evaluation_mode":"sampled","online_sampling_rate":0.1}\'')
+@click.option("--advanced-settings", "advanced_settings", default=None, help='JSON object overriding the judge\'s online-evaluation configuration. All four fields are required when this flag is supplied.\n\n\x08\nShape:\n  {\n    "online_evaluation_mode": "continuous" | "on_demand",\n    "online_sampling_rate": <number 0-100>,\n    "online_span_triggers": [\n      {"field":"span_name"|"span_attribute","operator":"contains"|"equals"|"exists","value":"<string>","key":"<attr-key>"?}\n    ],\n    "online_session_scoring": <bool>\n  }\n\n"continuous" runs the judge automatically on qualifying spans; "on_demand" requires a manual `judgment traces evaluate` call. online_sampling_rate is a percent of matching spans to score.\n\nExample: --advanced-settings \'{"online_evaluation_mode":"continuous","online_sampling_rate":10,"online_span_triggers":[],"online_session_scoring":false}\'')
 @click.option("--judge-id", "judge_id", default=None, help='Attach the new behavior to an existing judge instead of creating one. The judge must be score_type=categorical and have no existing behaviors.')
 @click.pass_context
 def behaviors_create_classifier(ctx, project_id, name, prompt, options, model, category_ids, advanced_settings, judge_id):
@@ -344,11 +344,11 @@ def judges_settings(ctx, project_id, judge_id):
 @click.argument("judge_id")
 @click.argument("evaluation_mode", type=click.Choice(['continuous', 'on_demand']))
 @click.argument("sampling_rate", type=float)
-@click.option("--span-triggers", "span_triggers", default=None, help='JSON array of span filters that restrict which spans the judge evaluates. Omit to evaluate all spans.\n\n\x08\nShape: [{"span_name":"<name>","attribute_filters":[{"key":"<attr>","op":"=","value":"<string>"}]}]\n\nExample: --span-triggers \'[{"span_name":"agent.run","attribute_filters":[]}]\'')
+@click.option("--span-triggers", "span_triggers", default=None, help='JSON array of span filters that restrict which spans the judge evaluates. Pass [] to evaluate all spans.\n\n\x08\nShape:\n  [\n    {\n      "field": "span_name" | "span_attribute",\n      "operator": "contains" | "equals" | "exists",\n      "value": "<string>",\n      "key": "<attribute key>"?\n    },\n    ...\n  ]\n\nUse "field":"span_name" to match on span names; "field":"span_attribute" with "key":"<attr>" to match on a span attribute\'s value. Triggers are ANDed together — a span must match every entry to be evaluated.\n\nExample: --span-triggers \'[{"field":"span_name","operator":"equals","value":"agent.run"}]\'')
 @click.option("--session-scoring", "session_scoring", default=None, type=bool, help='When true, run the judge at session granularity instead of per-span.')
 @click.pass_context
 def judges_update_settings(ctx, project_id, judge_id, evaluation_mode, sampling_rate, span_triggers, session_scoring):
-    'Update a judge\'s online-evaluation configuration.\n\n\x08\nApplies to all behaviors backed by the judge. evaluation_mode selects whether the judge runs continuously or on demand; sampling_rate is a percentage (0-100) of qualifying spans to score. Requires the developer role.\n\n\x08\n\x08\nExample: judgment judges update-settings <PROJECT_ID> <JUDGE_ID> sampled 10 \\\n  --span-triggers \'[{"span_name":"agent.run","attribute_filters":[]}]\''
+    'Update a judge\'s online-evaluation configuration.\n\n\x08\nApplies to all behaviors backed by the judge. EVALUATION_MODE selects whether the judge runs continuously on every matching span ("continuous") or only when explicitly invoked ("on_demand"); SAMPLING_RATE is a percentage (0-100) of qualifying spans to score. Requires the developer role.\n\n\x08\n\x08\nExample: judgment judges update-settings <PROJECT_ID> <JUDGE_ID> continuous 10 \\\n  --span-triggers \'[{"field":"span_name","operator":"equals","value":"agent.run"}]\''
     url = "/judges/update-settings"
     body = {}
     body["project_id"] = project_id

From 97bdd70997ed2951460bddf26074c65c7e381c3e Mon Sep 17 00:00:00 2001
From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Date: Fri, 24 Apr 2026 00:41:07 +0000
Subject: [PATCH 3/6] Remove misleading 'pass null to clear' note from
 behaviors update help

Co-Authored-By: Aaryan Divate <divateaaryan@gmail.com>
---
 scripts/help_overrides.py              | 2 +-
 src/judgment_cli/generated_commands.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/help_overrides.py b/scripts/help_overrides.py
index b7a76b1..02cb333 100644
--- a/scripts/help_overrides.py
+++ b/scripts/help_overrides.py
@@ -392,7 +392,7 @@
         "judge_id": "Attach the new behavior to an existing judge instead of creating one. The judge must be score_type=categorical and have no existing behaviors.",
     },
     "behaviors.update": {
-        "description": "New human-readable description for the behavior. Pass null to clear it.",
+        "description": "New human-readable description for the behavior.",
     },
     "behaviors.delete": {
         "delete_scorer": "When true, also delete the underlying prompt scorer if no other behaviors reference it.",
diff --git a/src/judgment_cli/generated_commands.py b/src/judgment_cli/generated_commands.py
index 1275f87..6895eca 100644
--- a/src/judgment_cli/generated_commands.py
+++ b/src/judgment_cli/generated_commands.py
@@ -264,7 +264,7 @@ def behaviors_list(ctx, project_id):
 @behaviors_group.command("update")
 @click.argument("project_id")
 @click.argument("behavior_id")
-@click.option("--description", "description", default=None, help='New human-readable description for the behavior. Pass null to clear it.')
+@click.option("--description", "description", default=None, help='New human-readable description for the behavior.')
 @click.pass_context
 def behaviors_update(ctx, project_id, behavior_id, description):
     "Update a behavior's description. Requires the developer role.\n\n\x08\n\x08\nExample: judgment behaviors update <PROJECT_ID> <BEHAVIOR_ID> --description 'New description'"

From 020a48dcc38e346ed0301917f59504797211afb3 Mon Sep 17 00:00:00 2001
From: Aaryan Divate <divateaaryan@gmail.com>
Date: Fri, 24 Apr 2026 14:47:50 -0700
Subject: [PATCH 4/6] Move descriptions to server and remove examples

---
 scripts/generate_cli.py                | 321 ++++++++++++-------
 scripts/help_overrides.py              | 421 -------------------------
 src/judgment_cli/generated_commands.py | 171 +++++-----
 src/judgment_cli/judges.py             |  26 +-
 src/judgment_cli/main.py               |  21 +-
 5 files changed, 323 insertions(+), 637 deletions(-)
 delete mode 100644 scripts/help_overrides.py

diff --git a/scripts/generate_cli.py b/scripts/generate_cli.py
index 52b3667..377add8 100644
--- a/scripts/generate_cli.py
+++ b/scripts/generate_cli.py
@@ -1,9 +1,25 @@
 #!/usr/bin/env python3
-"""
-Auto-generate Click CLI commands from the Judgment OpenAPI spec.
+"""Auto-generate Click CLI commands from the Judgment OpenAPI spec.
+
+This script consumes ``cli-server``'s OpenAPI document and emits
+``src/judgment_cli/generated_commands.py``. The CLI server is the single
+source of truth for command names, descriptions, option help, and group
+structure — this generator is a thin renderer.
+
+The generator reads, in order of preference:
+
+* ``operationId``  — required, ``"<group>.<command>"``. Determines both the
+  Click group and the command name.
+* ``tags[0]``      — used as a fallback for the group when ``operationId``
+  is missing.
+* spec-level ``tags`` — used to populate group docstrings.
+* operation ``summary`` + ``description`` — Click command docstring.
+* schema-level ``description`` on each request-body / query property —
+  Click ``--option`` help.
 
-Reads the OpenAPI spec and produces src/judgment_cli/generated_commands.py
-containing Click command groups for every operation except excluded endpoints.
+Routes whose ``operationId`` is in :data:`MANUAL_COMMANDS` are skipped so
+that hand-written commands (e.g. ``judgment judges upload``) own those
+slots.
 
 Run ``python scripts/generate_cli.py --help`` for usage.
 """
@@ -20,25 +36,15 @@
 
 import httpx
 
-from help_overrides import command_help, option_help
-
 DEFAULT_SPEC = "https://cli.judgmentlabs.ai/openapi/json"
-EXCLUDED_PATHS = {"/", "/health/", "/judges/upload"}
-
-GROUP_DESCRIPTIONS: dict[str, str] = {
-    "auth": "Authentication and user information",
-    "automations": "Manage automations",
-    "docs": "Search and read docs",
-    "projects": "Manage projects",
-    "root": "Top-level API routes",
-    "traces": "View and manage traces",
-    "judges": "Manage judges (scorers)",
-    "behaviors": "View and manage behaviors",
-    "sessions": "View and manage sessions",
-}
+
+# Operations whose CLI command is hand-written in judgment_cli/judges.py
+# (or another extension module) and must not be auto-generated.
+MANUAL_COMMANDS = {"judges.upload"}
+
 
 # ---------------------------------------------------------------------------
-# Helpers
+# Spec loading + traversal
 # ---------------------------------------------------------------------------
 
 
@@ -51,58 +57,64 @@ def load_spec(source: str) -> dict:
         return json.load(f)
 
 
-def derive_group_and_command(path: str, method: str) -> tuple[str, str]:
-    parts = [part for part in path.strip("/").split("/") if part]
-    if not parts:
-        return "root", "index"
-    if len(parts) == 1:
-        if method == "GET":
-            return parts[0], "list"
-        return parts[0], method.lower()
-
-    group = parts[0]
-    tail = parts[-1]
-    if tail == "detail":
-        command = "get"
-    elif tail == "page":
-        command = "read"
-    else:
-        command = tail
-    return group, command
-
+def derive_group_and_command(
+    operation: dict, path: str, method: str
+) -> tuple[str, str]:
+    """Map an OpenAPI operation to ``(group, command)``.
+
+    Requires an explicit ``operationId`` of the form ``"<group>.<command>"``.
+    Falls back to ``tags[0]`` plus the path's last segment when ``operationId``
+    is missing but a tag is present. Raises ``SystemExit`` if neither is set —
+    every route in cli-server is expected to declare both.
+    """
+    op_id = operation.get("operationId")
+    if op_id and "." in op_id:
+        group, _, command = op_id.partition(".")
+        return group, command
+
+    tags = operation.get("tags") or []
+    if tags:
+        group = str(tags[0])
+        parts = [p for p in path.strip("/").split("/") if p and p != group]
+        command = parts[-1] if parts else (
+            "list" if method.upper() == "GET" else method.lower()
+        )
+        return group, command
 
-def derive_description(operation: dict, group: str, command: str) -> str:
-    override = command_help(group, command)
-    if override:
-        return override
-    summary = operation.get("summary")
-    description = operation.get("description")
-    if summary and description:
-        return f"{summary}\n\n{description}"
-    return summary or description or f"{command.replace('-', ' ').title()} {group}"
+    raise SystemExit(
+        f"Operation {method} {path} has neither an operationId nor a tag. "
+        "Add `detail: { tags: ['<group>'], operationId: '<group>.<command>' }` "
+        "to the route in the server."
+    )
 
 
 def collect_operations(spec: dict) -> list[dict[str, Any]]:
     operations: list[dict[str, Any]] = []
     seen: set[tuple[str, str]] = set()
     for path, path_item in spec.get("paths", {}).items():
-        if path in EXCLUDED_PATHS:
+        if not isinstance(path_item, dict):
             continue
         for method, operation in path_item.items():
+            if method.upper() not in {"GET", "POST", "PUT", "PATCH", "DELETE"}:
+                continue
             if not isinstance(operation, dict):
                 continue
-            group, command = derive_group_and_command(path, method.upper())
-            original_command = command
+            op_id = operation.get("operationId")
+            if op_id in MANUAL_COMMANDS:
+                continue
+            group, command = derive_group_and_command(
+                operation, path, method.upper()
+            )
+            base = command
             suffix = 2
             while (group, command) in seen:
-                command = f"{original_command}-{suffix}"
+                command = f"{base}-{suffix}"
                 suffix += 1
             seen.add((group, command))
             operations.append(
                 {
                     "group": group,
                     "command": command,
-                    "description": derive_description(operation, group, command),
                     "path": path,
                     "method": method.upper(),
                     "operation": operation,
@@ -112,32 +124,48 @@ def collect_operations(spec: dict) -> list[dict[str, Any]]:
     return operations
 
 
+def collect_group_descriptions(spec: dict) -> dict[str, str]:
+    """Pull `{tag_name: description}` out of the spec's top-level tags array."""
+    descriptions: dict[str, str] = {}
+    for tag in spec.get("tags") or []:
+        if not isinstance(tag, dict):
+            continue
+        name = tag.get("name")
+        desc = tag.get("description")
+        if name and desc:
+            descriptions[name] = desc
+    return descriptions
+
+
+# ---------------------------------------------------------------------------
+# Schema + parameter introspection
+# ---------------------------------------------------------------------------
+
+
 def extract_path_params(path: str) -> list[str]:
     return re.findall(r"\{(\w+)\}", path)
 
 
 def extract_query_params(operation: dict) -> list[dict]:
-    params = []
-    for p in operation.get("parameters", []):
-        if p.get("in") == "query":
-            params.append(
-                {
-                    "name": p["name"],
-                    "required": p.get("required", False),
-                    "schema": p.get("schema", {}),
-                }
-            )
-    return params
+    return [
+        {
+            "name": p["name"],
+            "required": p.get("required", False),
+            "schema": p.get("schema", {}),
+        }
+        for p in operation.get("parameters", [])
+        if p.get("in") == "query"
+    ]
 
 
 def cli_option_name(name: str) -> str:
-    """Convert a camelCase or snake_case name to a CLI --option-name."""
+    """Convert ``camelCase`` or ``snake_case`` to a kebab CLI flag name."""
     s = re.sub(r"([a-z0-9])([A-Z])", r"\1_\2", name)
     return s.lower().replace("_", "-")
 
 
 def py_var_name(name: str) -> str:
-    """Ensure a valid Python variable name."""
+    """Coerce *name* into a valid (non-reserved) Python identifier."""
     s = re.sub(r"([a-z0-9])([A-Z])", r"\1_\2", name)
     s = s.lower().replace("-", "_")
     if keyword.iskeyword(s):
@@ -170,6 +198,18 @@ def _is_scalar_array_schema(schema: dict[str, Any]) -> bool:
     return bool(item_schema and _is_scalar_schema(item_schema))
 
 
+def _is_positional_scalar(schema: dict[str, Any]) -> bool:
+    """Whether a required scalar should render as a Click positional arg.
+
+    Only plain string scalars (no enum) qualify — they typically identify
+    entities like ``PROJECT_ID`` or ``NAME`` and read naturally positionally.
+    Required enums, numbers, and booleans become required ``--option`` flags
+    instead, where a name like ``--combine-type all`` reads better than an
+    anonymous ``{all|any}`` slot in the signature.
+    """
+    return _schema_type(schema) == "string" and not schema.get("enum")
+
+
 def click_type_expr(schema: dict[str, Any]) -> str | None:
     schema_type = _schema_type(schema)
     if schema_type == "integer":
@@ -190,7 +230,7 @@ def click_choice_expr(schema: dict[str, Any]) -> str | None:
 
 
 def _schema_description(schema: dict[str, Any]) -> str | None:
-    """Pull a human-readable description out of an OpenAPI schema, if any."""
+    """Extract a human description from a schema (or any of its anyOf branches)."""
     desc = schema.get("description")
     if desc:
         return desc
@@ -200,17 +240,14 @@ def _schema_description(schema: dict[str, Any]) -> str | None:
     return None
 
 
-def _quote_help(text: str) -> str:
-    """Encode a string as a Python source literal."""
+def _quote(text: str) -> str:
     return repr(text)
 
 
 def _emit_docstring(description: str) -> str:
-    """Build the docstring source line for a generated command."""
+    """Render a Click command docstring, preserving paragraph breaks."""
     short, _, long = description.partition("\n\n")
     if not long:
-        if not short.endswith((".", "!", "?")):
-            short += "."
         return f'    """{short}"""'
     paragraphs = ["\b\n" + p for p in long.split("\n\n")]
     full = short + "\n\n" + "\n\n".join(paragraphs)
@@ -221,8 +258,8 @@ def click_param_args(
     schema: dict[str, Any],
     *,
     include_help: bool = False,
-    help_override: str | None = None,
 ) -> str:
+    """Render extra ``click.option``/``click.argument`` args (type, choices, help)."""
     args: list[str] = []
     choice_expr = click_choice_expr(schema)
     if choice_expr:
@@ -232,33 +269,37 @@ def click_param_args(
         if type_expr:
             args.append(f"type={type_expr}")
     if include_help:
-        desc = help_override or _schema_description(schema)
+        desc = _schema_description(schema)
         if desc:
-            args.append(f"help={_quote_help(desc)}")
+            args.append(f"help={_quote(desc)}")
     return f", {', '.join(args)}" if args else ""
 
 
 def extract_json_body_properties(operation: dict) -> list[dict[str, Any]]:
     request_body = operation.get("requestBody") or {}
-    content = request_body.get("content") or {}
-    json_content = content.get("application/json") or {}
+    json_content = (request_body.get("content") or {}).get(
+        "application/json"
+    ) or {}
     schema = json_content.get("schema") or {}
     if schema.get("type") != "object":
         return []
 
     required = set(schema.get("required", []))
-    props: list[dict[str, Any]] = []
-    for name, prop_schema in (schema.get("properties") or {}).items():
-        props.append(
-            {
-                "name": name,
-                "required": name in required,
-                "schema": prop_schema,
-                "scalar": _is_scalar_schema(prop_schema),
-                "scalar_array": _is_scalar_array_schema(prop_schema),
-            }
-        )
-    return props
+    return [
+        {
+            "name": name,
+            "required": name in required,
+            "schema": prop_schema,
+            "scalar": _is_scalar_schema(prop_schema),
+            "scalar_array": _is_scalar_array_schema(prop_schema),
+            "positional": (
+                name in required
+                and _is_scalar_schema(prop_schema)
+                and _is_positional_scalar(prop_schema)
+            ),
+        }
+        for name, prop_schema in (schema.get("properties") or {}).items()
+    ]
 
 
 # ---------------------------------------------------------------------------
@@ -275,14 +316,12 @@ def generate_command_code(
     method: str,
     operation: dict,
 ) -> list[str]:
-    """Generate the source lines for a single Click command."""
+    """Render the source lines for a single Click command."""
     path_params = extract_path_params(path)
     query_params = extract_query_params(operation)
     body_props = extract_json_body_properties(operation)
 
-    lines: list[str] = []
-
-    lines.append(f'@{group_name}_group.command("{cmd_name}")')
+    lines: list[str] = [f'@{group_name}_group.command("{cmd_name}")']
 
     for pp in path_params:
         lines.append(f'@click.argument("{pp}")')
@@ -290,36 +329,57 @@ def generate_command_code(
     for qp in query_params:
         opt = cli_option_name(qp["name"])
         var = py_var_name(qp["name"])
-        override = option_help(group_name, cmd_name, qp["name"])
-        if qp["required"]:
+        if qp["required"] and _is_positional_scalar(qp["schema"]):
             type_arg = click_param_args(qp["schema"])
             lines.append(f'@click.argument("{qp["name"]}"{type_arg})')
+        elif qp["required"]:
+            type_arg = click_param_args(qp["schema"], include_help=True)
+            lines.append(
+                f'@click.option("--{opt}", "{var}", required=True{type_arg})'
+            )
         else:
-            type_arg = click_param_args(qp["schema"], include_help=True, help_override=override)
-            lines.append(f'@click.option("--{opt}", "{var}", default=None{type_arg})')
+            type_arg = click_param_args(qp["schema"], include_help=True)
+            lines.append(
+                f'@click.option("--{opt}", "{var}", default=None{type_arg})'
+            )
 
     if method != "GET":
         for prop in body_props:
             opt = cli_option_name(prop["name"])
             var = py_var_name(prop["name"])
-            override = option_help(group_name, cmd_name, prop["name"])
-            if prop["scalar"] and prop["required"]:
+            if prop["positional"]:
                 type_arg = click_param_args(prop["schema"])
                 lines.append(f'@click.argument("{prop["name"]}"{type_arg})')
             elif prop["scalar"]:
-                type_arg = click_param_args(prop["schema"], include_help=True, help_override=override)
-                lines.append(f'@click.option("--{opt}", "{var}", default=None{type_arg})')
+                type_arg = click_param_args(prop["schema"], include_help=True)
+                required_arg = (
+                    ", required=True" if prop["required"] else ", default=None"
+                )
+                lines.append(
+                    f'@click.option("--{opt}", "{var}"{required_arg}{type_arg})'
+                )
             elif prop["scalar_array"]:
-                item_type_arg = click_param_args(_array_item_schema(prop["schema"]) or {}, include_help=False)
-                desc = override or _schema_description(prop["schema"]) or f"Repeatable; one value per item for {prop['name']}."
+                item_type_arg = click_param_args(
+                    _array_item_schema(prop["schema"]) or {}, include_help=False
+                )
+                desc = (
+                    _schema_description(prop["schema"])
+                    or f"Repeatable; one value per item for {prop['name']}."
+                )
+                required_arg = ", required=True" if prop["required"] else ""
                 lines.append(
-                    f'@click.option("--{opt}", "{var}", multiple=True{item_type_arg}, help={_quote_help(desc)})'
+                    f'@click.option("--{opt}", "{var}", multiple=True{required_arg}{item_type_arg}, help={_quote(desc)})'
                 )
             else:
-                desc = override or _schema_description(prop["schema"]) or f"JSON value for {prop['name']}."
-                required_arg = ", required=True" if prop["required"] else ", default=None"
+                desc = (
+                    _schema_description(prop["schema"])
+                    or f"JSON value for {prop['name']}."
+                )
+                required_arg = (
+                    ", required=True" if prop["required"] else ", default=None"
+                )
                 lines.append(
-                    f'@click.option("--{opt}", "{var}"{required_arg}, help={_quote_help(desc)})'
+                    f'@click.option("--{opt}", "{var}"{required_arg}, help={_quote(desc)})'
                 )
 
     lines.append("@click.pass_context")
@@ -349,7 +409,9 @@ def generate_command_code(
         call_args = [f'"{method}"', "url"]
         if query_params:
             call_args.append("params=params")
-        lines.append(f'    result = ctx.obj["client"].request({", ".join(call_args)})')
+        lines.append(
+            f'    result = ctx.obj["client"].request({", ".join(call_args)})'
+        )
         lines.append("    _output(result)")
         return lines
 
@@ -372,21 +434,29 @@ def generate_command_code(
                 lines.append(f"    if {var} is not None:")
                 lines.append(f'        body["{prop["name"]}"] = json.loads({var})')
 
-    lines.append(f'    result = ctx.obj["client"].request("{method}", url, json_body=body)')
+    lines.append(
+        f'    result = ctx.obj["client"].request("{method}", url, json_body=body)'
+    )
     lines.append("    _output(result)")
 
     return lines
 
 
+def derive_description(operation: dict, group: str, command: str) -> str:
+    summary = operation.get("summary")
+    description = operation.get("description")
+    if summary and description:
+        return f"{summary}\n\n{description}"
+    return summary or description or f"{command.replace('-', ' ').title()} {group}"
+
+
 def generate_all(spec: dict) -> str:
-    """Generate the full generated_commands.py source."""
     operations = collect_operations(spec)
+    group_descriptions = collect_group_descriptions(spec)
     groups: dict[str, list[dict]] = {}
     for entry in operations:
-        gname = entry["group"]
-        groups.setdefault(gname, []).append(entry)
+        groups.setdefault(entry["group"], []).append(entry)
 
-    # --- header ---
     out = textwrap.dedent("""\
         # Auto-generated by scripts/generate_cli.py
         # DO NOT EDIT MANUALLY
@@ -401,9 +471,16 @@ def generate_all(spec: dict) -> str:
 
     """)
 
-    # --- groups + commands ---
+    missing = sorted(set(groups) - set(group_descriptions))
+    if missing:
+        raise SystemExit(
+            "Missing top-level OpenAPI tag description for group(s): "
+            f"{', '.join(missing)}. Add a `{{ name, description }}` entry to "
+            "the `tags` array in cli-server/index.ts for each group."
+        )
+
     for gname in sorted(groups):
-        desc = GROUP_DESCRIPTIONS.get(gname, f"Manage {gname}")
+        desc = group_descriptions[gname]
         out += "\n"
         out += f"# {'─' * 68}\n"
         out += f"# Group: {gname}\n"
@@ -411,8 +488,7 @@ def generate_all(spec: dict) -> str:
         out += "\n\n"
         out += f'@click.group("{gname}")\n'
         out += f"def {gname}_group() -> None:\n"
-        out += f'    """{desc}."""\n'
-        out += "    pass\n"
+        out += f"    {_quote(desc)}\n"
 
         for entry in groups[gname]:
             out += "\n\n"
@@ -421,20 +497,22 @@ def generate_all(spec: dict) -> str:
                 group_name=gname,
                 cmd_name=entry["command"],
                 func_name=func_name,
-                description=entry["description"],
+                description=derive_description(
+                    entry["operation"], gname, entry["command"]
+                ),
                 path=entry["path"],
                 method=entry["method"],
                 operation=entry["operation"],
             )
             out += "\n".join(cmd_lines) + "\n"
 
-    # --- register function ---
-    out += "\n\n"
+        out += "\n"
+
+    out += "\n"
     out += "def register_commands(cli: click.Group) -> None:\n"
     out += '    """Register all generated command groups on the root CLI."""\n'
     for gname in sorted(groups):
         out += f'    cli.add_command({gname}_group, "{gname}")\n'
-    out += ""
 
     return out
 
@@ -443,6 +521,7 @@ def generate_all(spec: dict) -> str:
 # Main
 # ---------------------------------------------------------------------------
 
+
 def main() -> None:
     parser = argparse.ArgumentParser(
         description="Generate Click CLI commands from the Judgment OpenAPI spec.",
diff --git a/scripts/help_overrides.py b/scripts/help_overrides.py
deleted file mode 100644
index 02cb333..0000000
--- a/scripts/help_overrides.py
+++ /dev/null
@@ -1,421 +0,0 @@
-"""Help text overrides for generated CLI commands.
-
-Entries are keyed by ``"<group>.<command>"``. ``COMMAND_HELP`` overrides the
-command docstring; ``OPTION_HELP`` overrides individual ``--flag`` help text.
-"""
-
-from __future__ import annotations
-
-
-_TRACE_FILTERS_HELP = """\
-Filter expressions, ANDed together. Each item is {"field":<field>,"op":<op>,"value":<value>}. Allowed ops depend on the field's type.
-
-\b
-Op groups:
-  STRING_OPS  = "=" | "!=" | "contains" | "does_not_contain" | "exists" | "is_absent"
-  NUMERIC_OPS = "=" | "!=" | "<" | "<=" | ">" | ">="
-  ARRAY_ANY   = "any"   (matches when the row's array overlaps the supplied values)
-
-String fields (op in STRING_OPS, value is a string): span_name, customer_id, customer_user_id, session_id, error, dataset_id.
-
-Numeric fields (op in NUMERIC_OPS, value is a number): duration (nanoseconds), llm_cost (USD).
-
-Array fields (op = "any", value is an array): tags (strings), rules_invoked (rule names from this project, strings), behaviors (behavior UUIDs).
-
-\b
-Special:
-  full_text_search       op = "contains", value is a string searched across span attribute text.
-  span_attributes_roots  matches a single span attribute key/value:
-                           {"field":"span_attributes_roots","key":"<attribute-name>","op":<STRING_OPS>,"value":"<string>"}
-
-Example: --filters '[{"field":"span_name","op":"=","value":"agent.run"}]'
-"""
-
-_SESSION_FILTERS_HELP = """\
-Filter expressions, ANDed together. Each item is {"field":<field>,"op":<op>,"value":<value>}. Allowed ops depend on the field's type.
-
-\b
-Op groups:
-  STRING_OPS  = "=" | "!=" | "contains" | "does_not_contain"
-  NUMERIC_OPS = "=" | "!=" | "<" | "<=" | ">" | ">="
-  ARRAY_ANY   = "any"   (matches when the row's array overlaps the supplied values)
-
-String fields (op in STRING_OPS, value is a string): session_id.
-
-Numeric fields (op in NUMERIC_OPS, value is a number): trace_count, latency (nanoseconds), total_cost (USD).
-
-Array fields (op = "any", value is an array): behaviors (behavior UUIDs).
-
-Example: --filters '[{"field":"trace_count","op":">=","value":5}]'
-"""
-
-_TIME_RANGE_HELP = (
-    '{"start_time":<iso8601-string>|null,"end_time":<iso8601-string>|null}. '
-    "Either bound may be null to leave that side open. "
-    "Invalid timestamps return 400."
-)
-
-_TRACE_SORT_BY_HELP = (
-    '{"field":<sort_field>,"direction":"asc"|"desc"} where sort_field is one of: '
-    "created_at, span_name, duration, llm_cost. "
-    'Default when omitted: {"field":"created_at","direction":"desc"}.'
-)
-
-_SESSION_SORT_BY_HELP = (
-    '{"field":<sort_field>,"direction":"asc"|"desc"} where sort_field is one of: '
-    "created_at, num_traces, latency, llm_cost. "
-    'Default when omitted: {"field":"created_at","direction":"desc"}.'
-)
-
-_TRACE_PAGINATION_HELP = """\
-{"limit":<int 1-200>,"cursorSortValue":<string>|null,"cursorItemId":<string>|null}.
-
-First page: pass null for both cursor fields. Each response returns nextCursor:{sort_value,trace_id} (or null when hasMore=false); copy those into cursorSortValue and cursorItemId for the next page.\
-"""
-
-_SESSION_PAGINATION_HELP = """\
-{"limit":<int 1-200>,"cursorSortValue":<string>|null,"cursorItemId":<string>|null}.
-
-First page: pass null for both cursor fields. Each response returns nextCursor:{sort_value,session_id} (or null when hasMore=false); copy those into cursorSortValue and cursorItemId for the next page.\
-"""
-
-
-_AUTOMATION_CONDITIONS_HELP = """\
-JSON array of rule conditions. Each condition references a named metric/scorer on the project and a comparison. Items are ANDed or ORed together based on the COMBINE_TYPE positional argument ("all" vs "any").
-
-\b
-Condition shape:
-  {
-    "metric": {
-      "scorer_type": "behavior" | "judge" | "prompt" | "custom" | "static" | "span_attribute" | "error",
-      "name": "<scorer or metric name>",
-      "threshold": <number | string | null>?
-    },
-    "comparison": "lt" | "gt" | "eq" | "gte" | "lte" | "fails" | "succeeds" | "chooses" | "detected" | "equals" | "contains" | "exists"
-  }
-
-\b
-Common scorer_type values:
-  behavior        Judge-scored behavior (name = behavior name, e.g. "Relevance")
-  static          Built-in metrics like "duration" (ms) or "llm_cost" (USD)
-  prompt/custom   Prompt or custom scorer by name
-  span_attribute  Arbitrary span attribute key (name = attribute key)
-  error           Span error condition
-
-Example: --conditions '[{"metric":{"scorer_type":"behavior","name":"Relevance","threshold":0.7},"comparison":"gte"}]'\
-"""
-
-_AUTOMATION_ACTIONS_HELP = """\
-JSON object describing what happens when the automation fires. All top-level keys are optional — include only the actions you want configured.
-
-\b
-Shape:
-  {
-    "notification": {
-      "enabled": <bool>?,
-      "communication_methods": ["email" | "slack" | "pagerduty"],
-      "email_addresses": ["<addr>", ...]?,
-      "pagerduty_config": {"routing_key":"<key>","severity":"critical"|"error"|"warning"|"info"}?
-    }?,
-    "dataset_addition": {
-      "enabled": <bool>?,
-      "dataset_name": "<dataset>",
-      "metadata_fields": <any>?
-    }?,
-    "behavior_evaluation": {
-      "enabled": <bool>?,
-      "behavior_judge_names": ["<judge_name>", ...]
-    }?
-  }
-
-Slack notifications are configured per-organization in the Judgment UI; pass "slack" in communication_methods to use them. Example: --actions '{"notification":{"communication_methods":["slack"]}}'\
-"""
-
-_AUTOMATION_TRIGGER_FREQUENCY_HELP = """\
-JSON 3-tuple [count, period, unit] describing the rate-limit window. Pass null to disable frequency limiting.
-
-\b
-Shape: [<max_trigger_count:number>, <period:number>, <unit:"seconds"|"minutes"|"hours"|"days">]
-
-Example: --trigger-frequency '[5, 1, "hours"]'  (max 5 triggers per 1 hour)\
-"""
-
-_AUTOMATION_COOLDOWN_HELP = """\
-JSON 2-tuple [period, unit] describing the minimum wait between triggers. Pass null to clear the cooldown.
-
-\b
-Shape: [<period:number>, <unit:"seconds"|"minutes"|"hours"|"days">]
-
-Example: --cooldown-period '[15, "minutes"]'  (at least 15 min between triggers)\
-"""
-
-_BEHAVIOR_ADVANCED_SETTINGS_HELP = """\
-JSON object overriding the judge's online-evaluation configuration. All four fields are required when this flag is supplied.
-
-\b
-Shape:
-  {
-    "online_evaluation_mode": "continuous" | "on_demand",
-    "online_sampling_rate": <number 0-100>,
-    "online_span_triggers": [
-      {"field":"span_name"|"span_attribute","operator":"contains"|"equals"|"exists","value":"<string>","key":"<attr-key>"?}
-    ],
-    "online_session_scoring": <bool>
-  }
-
-"continuous" runs the judge automatically on qualifying spans; "on_demand" requires a manual `judgment traces evaluate` call. online_sampling_rate is a percent of matching spans to score.
-
-Example: --advanced-settings '{"online_evaluation_mode":"continuous","online_sampling_rate":10,"online_span_triggers":[],"online_session_scoring":false}'\
-"""
-
-_BEHAVIOR_CATEGORY_IDS_HELP = (
-    "UUID of a category to attach the behavior to. Repeat the flag to attach "
-    "multiple categories: --category-ids <uuid1> --category-ids <uuid2>."
-)
-
-_CLASSIFIER_OPTIONS_HELP = """\
-JSON array of the allowed output categories the classifier judge can return. Must contain at least one option.
-
-\b
-Shape:
-  [
-    {"name":"<label>", "description":"<optional human description>", "category_ids":["<uuid>", ...]},
-    ...
-  ]
-
-Example: --options '[{"name":"Good","description":"helpful response"},{"name":"Bad","description":"unhelpful or incorrect"}]'\
-"""
-
-_JUDGE_SPAN_TRIGGERS_HELP = """\
-JSON array of span filters that restrict which spans the judge evaluates. Pass [] to evaluate all spans.
-
-\b
-Shape:
-  [
-    {
-      "field": "span_name" | "span_attribute",
-      "operator": "contains" | "equals" | "exists",
-      "value": "<string>",
-      "key": "<attribute key>"?
-    },
-    ...
-  ]
-
-Use "field":"span_name" to match on span names; "field":"span_attribute" with "key":"<attr>" to match on a span attribute's value. Triggers are ANDed together — a span must match every entry to be evaluated.
-
-Example: --span-triggers '[{"field":"span_name","operator":"equals","value":"agent.run"}]'\
-"""
-
-
-COMMAND_HELP: dict[str, str] = {
-    "traces.search": (
-        "Search traces in a project.\n"
-        "\n"
-        "Filtering, sorting, time bounds, and pagination are passed as JSON via the body fields below — see each flag's description for its full reference.\n"
-        "\n"
-        "\b\n"
-        'Example: judgment traces search <PROJECT_ID> \\\n'
-        "  --filters '[{\"field\":\"span_name\",\"op\":\"=\",\"value\":\"agent.run\"}]' \\\n"
-        '  --sort-by \'{"field":"llm_cost","direction":"desc"}\' \\\n'
-        '  --pagination \'{"limit":25,"cursorSortValue":null,"cursorItemId":null}\''
-    ),
-    "sessions.search": (
-        "Search sessions in a project.\n"
-        "\n"
-        "Filtering, sorting, time bounds, and pagination are passed as JSON via the body fields below — see each flag's description for its full reference.\n"
-        "\n"
-        "\b\n"
-        'Example: judgment sessions search <PROJECT_ID> \\\n'
-        "  --filters '[{\"field\":\"trace_count\",\"op\":\">=\",\"value\":5}]' \\\n"
-        '  --sort-by \'{"field":"latency","direction":"desc"}\' \\\n'
-        '  --pagination \'{"limit":25,"cursorSortValue":null,"cursorItemId":null}\''
-    ),
-    "automations.create": (
-        "Create an automation (rule) in a project.\n"
-        "\n"
-        "An automation watches behavior/latency/cost metrics on the project and fires actions when its conditions match. Requires the developer role.\n"
-        "\n"
-        "\b\n"
-        "Example: judgment automations create <PROJECT_ID> my-rule any \\\n"
-        "  --description 'Alert on low relevance scores' \\\n"
-        "  --conditions '[{\"metric\":{\"scorer_type\":\"behavior\",\"name\":\"Relevance\",\"threshold\":0.7},\"comparison\":\"lt\"}]' \\\n"
-        "  --actions '{\"notification\":{\"communication_methods\":[\"slack\"]}}' \\\n"
-        "  --cooldown-period 5 --cooldown-period-unit minutes"
-    ),
-    "automations.update": (
-        "Update an existing automation. All fields other than the positional arguments are optional — only supplied flags are applied.\n"
-        "\n"
-        "Use --active true/false to enable or disable without changing other fields. Requires the developer role.\n"
-        "\n"
-        "\b\n"
-        "Example: judgment automations update <PROJECT_ID> <RULE_ID> --active false"
-    ),
-    "automations.delete": (
-        "Delete an automation. Requires the admin role.\n"
-        "\n"
-        "\b\n"
-        "Example: judgment automations delete <PROJECT_ID> <RULE_ID>"
-    ),
-    "automations.get": (
-        "Fetch the full definition of a single automation by ID.\n"
-        "\n"
-        "\b\n"
-        "Example: judgment automations get <PROJECT_ID> <RULE_ID>"
-    ),
-    "behaviors.create-binary": (
-        "Create a binary (Yes/No) behavior backed by a prompt judge.\n"
-        "\n"
-        'Scores each trace/span as "Yes" (true) or "No" (false) using the prompt you supply. Pass --judge-id to attach to an existing judge instead of creating a new prompt scorer. Requires the developer role.\n'
-        "\n"
-        "\b\n"
-        "Example: judgment behaviors create-binary <PROJECT_ID> 'Relevance' \\\n"
-        "  'Was the assistant response relevant to the user message?' \\\n"
-        "  --description 'Binary relevance check' --model gpt-5.2"
-    ),
-    "behaviors.create-classifier": (
-        "Create a categorical (multi-class) behavior backed by a prompt judge.\n"
-        "\n"
-        "Scores each trace/span into one of the --options categories. Pass --judge-id to attach to an existing classifier judge instead of creating a new one. Requires the developer role.\n"
-        "\n"
-        "\b\n"
-        "Example: judgment behaviors create-classifier <PROJECT_ID> 'Sentiment' \\\n"
-        "  'Classify the assistant response tone.' \\\n"
-        "  --options '[{\"name\":\"Positive\"},{\"name\":\"Neutral\"},{\"name\":\"Negative\"}]'"
-    ),
-    "behaviors.update": (
-        "Update a behavior's description. Requires the developer role.\n"
-        "\n"
-        "\b\n"
-        "Example: judgment behaviors update <PROJECT_ID> <BEHAVIOR_ID> --description 'New description'"
-    ),
-    "behaviors.delete": (
-        "Delete a behavior. Requires the admin role.\n"
-        "\n"
-        "Binary behaviors always delete both the true and false rows. For classifier behaviors, pass --delete-all-values true to delete every category row for the judge at once. Pass --delete-scorer true to also delete the underlying prompt scorer when no other behaviors reference it.\n"
-        "\n"
-        "\b\n"
-        "Example: judgment behaviors delete <PROJECT_ID> <BEHAVIOR_ID> --delete-scorer true"
-    ),
-    "judges.update-settings": (
-        "Update a judge's online-evaluation configuration.\n"
-        "\n"
-        "Applies to all behaviors backed by the judge. EVALUATION_MODE selects whether the judge runs continuously on every matching span (\"continuous\") or only when explicitly invoked (\"on_demand\"); SAMPLING_RATE is a percentage (0-100) of qualifying spans to score. Requires the developer role.\n"
-        "\n"
-        "\b\n"
-        "Example: judgment judges update-settings <PROJECT_ID> <JUDGE_ID> continuous 10 \\\n"
-        "  --span-triggers '[{\"field\":\"span_name\",\"operator\":\"equals\",\"value\":\"agent.run\"}]'"
-    ),
-    "projects.create": (
-        "Create a new project in your organization. Requires the developer role.\n"
-        "\n"
-        "\b\n"
-        "Example: judgment projects create my-new-project"
-    ),
-    "projects.add-favorite": (
-        "Mark a project as a favorite for your user so it appears pinned in the UI.\n"
-        "\n"
-        "\b\n"
-        "Example: judgment projects add-favorite <PROJECT_ID>"
-    ),
-    "projects.remove-favorite": (
-        "Remove a project from your user's favorites.\n"
-        "\n"
-        "\b\n"
-        "Example: judgment projects remove-favorite <PROJECT_ID>"
-    ),
-    "traces.add-tags": (
-        "Attach one or more string tags to an existing trace.\n"
-        "\n"
-        "Tags are additive — existing tags on the trace are preserved. Repeat --tags to add multiple tags.\n"
-        "\n"
-        "\b\n"
-        "Example: judgment traces add-tags <PROJECT_ID> <TRACE_ID> \\\n"
-        "  --tags regression --tags investigate"
-    ),
-    "traces.evaluate": (
-        "Queue traces for re-evaluation by the project's judges.\n"
-        "\n"
-        "Pass specific trace IDs with --trace-ids (repeat the flag) or use --evaluate-all true to re-evaluate every trace in the project. Use --specific-judge-names to restrict which judges run (repeat the flag). Requires the developer role.\n"
-        "\n"
-        "\b\n"
-        "Example: judgment traces evaluate <PROJECT_ID> \\\n"
-        "  --trace-ids <TRACE_ID_1> --trace-ids <TRACE_ID_2> \\\n"
-        "  --specific-judge-names Relevance"
-    ),
-}
-
-
-OPTION_HELP: dict[str, dict[str, str]] = {
-    "traces.search": {
-        "filters": _TRACE_FILTERS_HELP,
-        "sort_by": _TRACE_SORT_BY_HELP,
-        "time_range": _TIME_RANGE_HELP,
-        "pagination": _TRACE_PAGINATION_HELP,
-    },
-    "sessions.search": {
-        "filters": _SESSION_FILTERS_HELP,
-        "sort_by": _SESSION_SORT_BY_HELP,
-        "time_range": _TIME_RANGE_HELP,
-        "pagination": _SESSION_PAGINATION_HELP,
-    },
-    "automations.create": {
-        "description": "Human-readable description shown in the UI.",
-        "conditions": _AUTOMATION_CONDITIONS_HELP,
-        "actions": _AUTOMATION_ACTIONS_HELP,
-        "cooldown_period": "Cooldown duration as a plain number. Paired with --cooldown-period-unit. Example: --cooldown-period 15 --cooldown-period-unit minutes.",
-        "cooldown_period_unit": "Unit for --cooldown-period: seconds | minutes | hours | days.",
-        "trigger_frequency_count": "Maximum triggers allowed within the window defined by --trigger-frequency-period and --trigger-frequency-period-unit.",
-        "trigger_frequency_period": "Window length for the trigger-frequency limit. Paired with --trigger-frequency-period-unit.",
-        "trigger_frequency_period_unit": "Unit for --trigger-frequency-period: seconds | minutes | hours | days.",
-    },
-    "automations.update": {
-        "name": "New name for the automation.",
-        "description": "New description for the automation.",
-        "conditions": _AUTOMATION_CONDITIONS_HELP,
-        "actions": _AUTOMATION_ACTIONS_HELP,
-        "active": "Enable (true) or disable (false) the automation without modifying other fields.",
-        "cooldown_period": _AUTOMATION_COOLDOWN_HELP,
-        "trigger_frequency": _AUTOMATION_TRIGGER_FREQUENCY_HELP,
-    },
-    "behaviors.create-binary": {
-        "description": "Human-readable description shown in the UI.",
-        "model": 'LLM model ID used by the judge prompt. Defaults to "gpt-5.2" when omitted.',
-        "category_ids": _BEHAVIOR_CATEGORY_IDS_HELP,
-        "advanced_settings": _BEHAVIOR_ADVANCED_SETTINGS_HELP,
-        "judge_id": "Attach the new behavior to an existing judge instead of creating one. The judge must be score_type=binary and have no existing behaviors.",
-    },
-    "behaviors.create-classifier": {
-        "options": _CLASSIFIER_OPTIONS_HELP,
-        "model": 'LLM model ID used by the judge prompt. Defaults to "gpt-5.2" when omitted.',
-        "category_ids": _BEHAVIOR_CATEGORY_IDS_HELP,
-        "advanced_settings": _BEHAVIOR_ADVANCED_SETTINGS_HELP,
-        "judge_id": "Attach the new behavior to an existing judge instead of creating one. The judge must be score_type=categorical and have no existing behaviors.",
-    },
-    "behaviors.update": {
-        "description": "New human-readable description for the behavior.",
-    },
-    "behaviors.delete": {
-        "delete_scorer": "When true, also delete the underlying prompt scorer if no other behaviors reference it.",
-        "delete_all_values": "For classifier behaviors, when true deletes every category row for this judge (not just the provided BEHAVIOR_ID). Ignored for binary behaviors (those always delete both rows).",
-    },
-    "judges.update-settings": {
-        "span_triggers": _JUDGE_SPAN_TRIGGERS_HELP,
-        "session_scoring": "When true, run the judge at session granularity instead of per-span.",
-    },
-    "traces.add-tags": {
-        "tags": "Tag string to attach to the trace. Repeat --tags to attach multiple tags: --tags a --tags b.",
-    },
-    "traces.evaluate": {
-        "evaluate_all": "When true, re-evaluate every trace in the project. Mutually exclusive with --trace-ids.",
-        "trace_ids": "Trace UUID to re-evaluate. Repeat --trace-ids to evaluate multiple: --trace-ids <id1> --trace-ids <id2>.",
-        "specific_judge_names": "Restrict evaluation to judges with these names. Repeat the flag to pass multiple. Omit to run every applicable judge.",
-    },
-}
-
-
-def command_help(group: str, command: str) -> str | None:
-    return COMMAND_HELP.get(f"{group}.{command}")
-
-
-def option_help(group: str, command: str, option: str) -> str | None:
-    return OPTION_HELP.get(f"{group}.{command}", {}).get(option)
diff --git a/src/judgment_cli/generated_commands.py b/src/judgment_cli/generated_commands.py
index 6895eca..02dadfa 100644
--- a/src/judgment_cli/generated_commands.py
+++ b/src/judgment_cli/generated_commands.py
@@ -17,25 +17,24 @@
 
 @click.group("automations")
 def automations_group() -> None:
-    """Manage automations."""
-    pass
+    'Manage automations (rules) that fire actions when metrics match conditions.'
 
 
 @automations_group.command("create")
 @click.argument("project_id")
 @click.argument("name")
 @click.option("--description", "description", default=None, help='Human-readable description shown in the UI.')
-@click.option("--conditions", "conditions", required=True, help='JSON array of rule conditions. Each condition references a named metric/scorer on the project and a comparison. Items are ANDed or ORed together based on the COMBINE_TYPE positional argument ("all" vs "any").\n\n\x08\nCondition shape:\n  {\n    "metric": {\n      "scorer_type": "behavior" | "judge" | "prompt" | "custom" | "static" | "span_attribute" | "error",\n      "name": "<scorer or metric name>",\n      "threshold": <number | string | null>?\n    },\n    "comparison": "lt" | "gt" | "eq" | "gte" | "lte" | "fails" | "succeeds" | "chooses" | "detected" | "equals" | "contains" | "exists"\n  }\n\n\x08\nCommon scorer_type values:\n  behavior        Judge-scored behavior (name = behavior name, e.g. "Relevance")\n  static          Built-in metrics like "duration" (ms) or "llm_cost" (USD)\n  prompt/custom   Prompt or custom scorer by name\n  span_attribute  Arbitrary span attribute key (name = attribute key)\n  error           Span error condition\n\nExample: --conditions \'[{"metric":{"scorer_type":"behavior","name":"Relevance","threshold":0.7},"comparison":"gte"}]\'')
-@click.argument("combine_type", type=click.Choice(['all', 'any']))
-@click.option("--actions", "actions", default=None, help='JSON object describing what happens when the automation fires. All top-level keys are optional — include only the actions you want configured.\n\n\x08\nShape:\n  {\n    "notification": {\n      "enabled": <bool>?,\n      "communication_methods": ["email" | "slack" | "pagerduty"],\n      "email_addresses": ["<addr>", ...]?,\n      "pagerduty_config": {"routing_key":"<key>","severity":"critical"|"error"|"warning"|"info"}?\n    }?,\n    "dataset_addition": {\n      "enabled": <bool>?,\n      "dataset_name": "<dataset>",\n      "metadata_fields": <any>?\n    }?,\n    "behavior_evaluation": {\n      "enabled": <bool>?,\n      "behavior_judge_names": ["<judge_name>", ...]\n    }?\n  }\n\nSlack notifications are configured per-organization in the Judgment UI; pass "slack" in communication_methods to use them. Example: --actions \'{"notification":{"communication_methods":["slack"]}}\'')
-@click.option("--cooldown-period", "cooldown_period", default=None, type=float, help='Cooldown duration as a plain number. Paired with --cooldown-period-unit. Example: --cooldown-period 15 --cooldown-period-unit minutes.')
-@click.option("--cooldown-period-unit", "cooldown_period_unit", default=None, type=click.Choice(['seconds', 'minutes', 'hours', 'days']), help='Unit for --cooldown-period: seconds | minutes | hours | days.')
-@click.option("--trigger-frequency-count", "trigger_frequency_count", default=None, type=float, help='Maximum triggers allowed within the window defined by --trigger-frequency-period and --trigger-frequency-period-unit.')
-@click.option("--trigger-frequency-period", "trigger_frequency_period", default=None, type=float, help='Window length for the trigger-frequency limit. Paired with --trigger-frequency-period-unit.')
-@click.option("--trigger-frequency-period-unit", "trigger_frequency_period_unit", default=None, type=click.Choice(['seconds', 'minutes', 'hours', 'days']), help='Unit for --trigger-frequency-period: seconds | minutes | hours | days.')
+@click.option("--conditions", "conditions", required=True, help='JSON array of rule conditions. Each condition references a named metric/scorer on the project and a comparison. Items are ANDed or ORed together based on `combine_type` (`all` vs `any`).\n\n**Condition shape:**\n```\n{\n  "metric": {\n    "scorer_type": "behavior" | "judge" | "prompt" | "custom" | "static" | "span_attribute" | "error",\n    "name": "<scorer or metric name>",\n    "threshold": <number | string | null>?\n  },\n  "comparison": "lt" | "gt" | "eq" | "gte" | "lte" | "fails" | "succeeds" | "chooses" | "detected" | "equals" | "contains" | "exists"\n}\n```\n\n**Common scorer_type values:**\n- `behavior` — Judge-scored behavior (name = behavior name, e.g. "Relevance")\n- `static` — Built-in metrics like "duration" (ms) or "llm_cost" (USD)\n- `prompt`/`custom` — Prompt or custom scorer by name\n- `span_attribute` — Arbitrary span attribute key (name = attribute key)\n- `error` — Span error condition')
+@click.option("--combine-type", "combine_type", required=True, type=click.Choice(['all', 'any']))
+@click.option("--actions", "actions", default=None, help='JSON object describing what happens when the automation fires. All top-level keys are optional — include only the actions you want configured.\n\n**Shape:**\n```\n{\n  "notification": {\n    "enabled": <bool>?,\n    "communication_methods": ["email" | "slack" | "pagerduty"],\n    "email_addresses": ["<addr>", ...]?,\n    "pagerduty_config": {"routing_key":"<key>","severity":"critical"|"error"|"warning"|"info"}?\n  }?,\n  "dataset_addition": {\n    "enabled": <bool>?,\n    "dataset_name": "<dataset>",\n    "metadata_fields": <any>?\n  }?,\n  "behavior_evaluation": {\n    "enabled": <bool>?,\n    "behavior_judge_names": ["<judge_name>", ...]\n  }?\n}\n```\n\nSlack notifications are configured per-organization in the Judgment UI; pass `"slack"` in `communication_methods` to use them.')
+@click.option("--cooldown-period", "cooldown_period", default=None, type=float, help='Cooldown duration as a plain number, paired with `cooldown_period_unit`. Example: 15 with unit `"minutes"`.')
+@click.option("--cooldown-period-unit", "cooldown_period_unit", default=None, type=click.Choice(['seconds', 'minutes', 'hours', 'days']))
+@click.option("--trigger-frequency-count", "trigger_frequency_count", default=None, type=float, help='Maximum triggers allowed within the trigger-frequency window.')
+@click.option("--trigger-frequency-period", "trigger_frequency_period", default=None, type=float, help='Window length for the trigger-frequency limit, paired with `trigger_frequency_period_unit`.')
+@click.option("--trigger-frequency-period-unit", "trigger_frequency_period_unit", default=None, type=click.Choice(['seconds', 'minutes', 'hours', 'days']))
 @click.pass_context
 def automations_create(ctx, project_id, name, description, conditions, combine_type, actions, cooldown_period, cooldown_period_unit, trigger_frequency_count, trigger_frequency_period, trigger_frequency_period_unit):
-    'Create an automation (rule) in a project.\n\n\x08\nAn automation watches behavior/latency/cost metrics on the project and fires actions when its conditions match. Requires the developer role.\n\n\x08\n\x08\nExample: judgment automations create <PROJECT_ID> my-rule any \\\n  --description \'Alert on low relevance scores\' \\\n  --conditions \'[{"metric":{"scorer_type":"behavior","name":"Relevance","threshold":0.7},"comparison":"lt"}]\' \\\n  --actions \'{"notification":{"communication_methods":["slack"]}}\' \\\n  --cooldown-period 5 --cooldown-period-unit minutes'
+    'Create an automation.\n\n\x08\nCreate an automation (rule) in a project. An automation watches behavior/latency/cost metrics and fires actions when its conditions match. Requires the developer role.'
     url = "/automations/create"
     body = {}
     body["project_id"] = project_id
@@ -65,7 +64,7 @@ def automations_create(ctx, project_id, name, description, conditions, combine_t
 @click.argument("rule_id")
 @click.pass_context
 def automations_delete(ctx, project_id, rule_id):
-    'Delete an automation. Requires the admin role.\n\n\x08\n\x08\nExample: judgment automations delete <PROJECT_ID> <RULE_ID>'
+    'Delete an automation.\n\n\x08\nDelete an automation. Requires the admin role.'
     url = "/automations/delete"
     body = {}
     body["project_id"] = project_id
@@ -79,7 +78,7 @@ def automations_delete(ctx, project_id, rule_id):
 @click.argument("rule_id")
 @click.pass_context
 def automations_get(ctx, project_id, rule_id):
-    'Fetch the full definition of a single automation by ID.\n\n\x08\n\x08\nExample: judgment automations get <PROJECT_ID> <RULE_ID>'
+    """Get an automation by ID."""
     url = "/automations/detail"
     body = {}
     body["project_id"] = project_id
@@ -105,15 +104,15 @@ def automations_list(ctx, project_id):
 @click.argument("rule_id")
 @click.option("--name", "name", default=None, help='New name for the automation.')
 @click.option("--description", "description", default=None, help='New description for the automation.')
-@click.option("--conditions", "conditions", default=None, help='JSON array of rule conditions. Each condition references a named metric/scorer on the project and a comparison. Items are ANDed or ORed together based on the COMBINE_TYPE positional argument ("all" vs "any").\n\n\x08\nCondition shape:\n  {\n    "metric": {\n      "scorer_type": "behavior" | "judge" | "prompt" | "custom" | "static" | "span_attribute" | "error",\n      "name": "<scorer or metric name>",\n      "threshold": <number | string | null>?\n    },\n    "comparison": "lt" | "gt" | "eq" | "gte" | "lte" | "fails" | "succeeds" | "chooses" | "detected" | "equals" | "contains" | "exists"\n  }\n\n\x08\nCommon scorer_type values:\n  behavior        Judge-scored behavior (name = behavior name, e.g. "Relevance")\n  static          Built-in metrics like "duration" (ms) or "llm_cost" (USD)\n  prompt/custom   Prompt or custom scorer by name\n  span_attribute  Arbitrary span attribute key (name = attribute key)\n  error           Span error condition\n\nExample: --conditions \'[{"metric":{"scorer_type":"behavior","name":"Relevance","threshold":0.7},"comparison":"gte"}]\'')
+@click.option("--conditions", "conditions", default=None, help='JSON array of rule conditions. Each condition references a named metric/scorer on the project and a comparison. Items are ANDed or ORed together based on `combine_type` (`all` vs `any`).\n\n**Condition shape:**\n```\n{\n  "metric": {\n    "scorer_type": "behavior" | "judge" | "prompt" | "custom" | "static" | "span_attribute" | "error",\n    "name": "<scorer or metric name>",\n    "threshold": <number | string | null>?\n  },\n  "comparison": "lt" | "gt" | "eq" | "gte" | "lte" | "fails" | "succeeds" | "chooses" | "detected" | "equals" | "contains" | "exists"\n}\n```\n\n**Common scorer_type values:**\n- `behavior` — Judge-scored behavior (name = behavior name, e.g. "Relevance")\n- `static` — Built-in metrics like "duration" (ms) or "llm_cost" (USD)\n- `prompt`/`custom` — Prompt or custom scorer by name\n- `span_attribute` — Arbitrary span attribute key (name = attribute key)\n- `error` — Span error condition')
 @click.option("--combine-type", "combine_type", default=None, type=click.Choice(['all', 'any']))
-@click.option("--actions", "actions", default=None, help='JSON object describing what happens when the automation fires. All top-level keys are optional — include only the actions you want configured.\n\n\x08\nShape:\n  {\n    "notification": {\n      "enabled": <bool>?,\n      "communication_methods": ["email" | "slack" | "pagerduty"],\n      "email_addresses": ["<addr>", ...]?,\n      "pagerduty_config": {"routing_key":"<key>","severity":"critical"|"error"|"warning"|"info"}?\n    }?,\n    "dataset_addition": {\n      "enabled": <bool>?,\n      "dataset_name": "<dataset>",\n      "metadata_fields": <any>?\n    }?,\n    "behavior_evaluation": {\n      "enabled": <bool>?,\n      "behavior_judge_names": ["<judge_name>", ...]\n    }?\n  }\n\nSlack notifications are configured per-organization in the Judgment UI; pass "slack" in communication_methods to use them. Example: --actions \'{"notification":{"communication_methods":["slack"]}}\'')
+@click.option("--actions", "actions", default=None, help='JSON object describing what happens when the automation fires. All top-level keys are optional — include only the actions you want configured.\n\n**Shape:**\n```\n{\n  "notification": {\n    "enabled": <bool>?,\n    "communication_methods": ["email" | "slack" | "pagerduty"],\n    "email_addresses": ["<addr>", ...]?,\n    "pagerduty_config": {"routing_key":"<key>","severity":"critical"|"error"|"warning"|"info"}?\n  }?,\n  "dataset_addition": {\n    "enabled": <bool>?,\n    "dataset_name": "<dataset>",\n    "metadata_fields": <any>?\n  }?,\n  "behavior_evaluation": {\n    "enabled": <bool>?,\n    "behavior_judge_names": ["<judge_name>", ...]\n  }?\n}\n```\n\nSlack notifications are configured per-organization in the Judgment UI; pass `"slack"` in `communication_methods` to use them.')
 @click.option("--active", "active", default=None, type=bool, help='Enable (true) or disable (false) the automation without modifying other fields.')
-@click.option("--cooldown-period", "cooldown_period", default=None, help='JSON 2-tuple [period, unit] describing the minimum wait between triggers. Pass null to clear the cooldown.\n\n\x08\nShape: [<period:number>, <unit:"seconds"|"minutes"|"hours"|"days">]\n\nExample: --cooldown-period \'[15, "minutes"]\'  (at least 15 min between triggers)')
-@click.option("--trigger-frequency", "trigger_frequency", default=None, help='JSON 3-tuple [count, period, unit] describing the rate-limit window. Pass null to disable frequency limiting.\n\n\x08\nShape: [<max_trigger_count:number>, <period:number>, <unit:"seconds"|"minutes"|"hours"|"days">]\n\nExample: --trigger-frequency \'[5, 1, "hours"]\'  (max 5 triggers per 1 hour)')
+@click.option("--cooldown-period", "cooldown_period", default=None, help='JSON 2-tuple `[period, unit]` describing the minimum wait between triggers. Pass null to clear the cooldown.\n\n**Shape:** `[<period:number>, <unit:"seconds"|"minutes"|"hours"|"days">]`\n\nExample: `[15, "minutes"]` (at least 15 min between triggers)')
+@click.option("--trigger-frequency", "trigger_frequency", default=None, help='JSON 3-tuple `[count, period, unit]` describing the rate-limit window. Pass null to disable frequency limiting.\n\n**Shape:** `[<max_trigger_count:number>, <period:number>, <unit:"seconds"|"minutes"|"hours"|"days">]`\n\nExample: `[5, 1, "hours"]` (max 5 triggers per 1 hour)')
 @click.pass_context
 def automations_update(ctx, project_id, rule_id, name, description, conditions, combine_type, actions, active, cooldown_period, trigger_frequency):
-    'Update an existing automation. All fields other than the positional arguments are optional — only supplied flags are applied.\n\n\x08\nUse --active true/false to enable or disable without changing other fields. Requires the developer role.\n\n\x08\n\x08\nExample: judgment automations update <PROJECT_ID> <RULE_ID> --active false'
+    'Update an automation.\n\n\x08\nUpdate an existing automation. All fields other than the IDs are optional — only supplied fields are applied. Use `active: true/false` to enable or disable without changing other fields. Requires the developer role.'
     url = "/automations/update"
     body = {}
     body["project_id"] = project_id
@@ -137,6 +136,7 @@ def automations_update(ctx, project_id, rule_id, name, description, conditions,
     result = ctx.obj["client"].request("POST", url, json_body=body)
     _output(result)
 
+
 # ────────────────────────────────────────────────────────────────────
 # Group: behaviors
 # ────────────────────────────────────────────────────────────────────
@@ -144,8 +144,7 @@ def automations_update(ctx, project_id, rule_id, name, description, conditions,
 
 @click.group("behaviors")
 def behaviors_group() -> None:
-    """View and manage behaviors."""
-    pass
+    'View and manage behaviors (the binary or categorical labels judges assign).'
 
 
 @behaviors_group.command("create-binary")
@@ -154,12 +153,12 @@ def behaviors_group() -> None:
 @click.argument("prompt")
 @click.option("--description", "description", default=None, help='Human-readable description shown in the UI.')
 @click.option("--model", "model", default=None, help='LLM model ID used by the judge prompt. Defaults to "gpt-5.2" when omitted.')
-@click.option("--category-ids", "category_ids", multiple=True, help='UUID of a category to attach the behavior to. Repeat the flag to attach multiple categories: --category-ids <uuid1> --category-ids <uuid2>.')
-@click.option("--advanced-settings", "advanced_settings", default=None, help='JSON object overriding the judge\'s online-evaluation configuration. All four fields are required when this flag is supplied.\n\n\x08\nShape:\n  {\n    "online_evaluation_mode": "continuous" | "on_demand",\n    "online_sampling_rate": <number 0-100>,\n    "online_span_triggers": [\n      {"field":"span_name"|"span_attribute","operator":"contains"|"equals"|"exists","value":"<string>","key":"<attr-key>"?}\n    ],\n    "online_session_scoring": <bool>\n  }\n\n"continuous" runs the judge automatically on qualifying spans; "on_demand" requires a manual `judgment traces evaluate` call. online_sampling_rate is a percent of matching spans to score.\n\nExample: --advanced-settings \'{"online_evaluation_mode":"continuous","online_sampling_rate":10,"online_span_triggers":[],"online_session_scoring":false}\'')
-@click.option("--judge-id", "judge_id", default=None, help='Attach the new behavior to an existing judge instead of creating one. The judge must be score_type=binary and have no existing behaviors.')
+@click.option("--category-ids", "category_ids", multiple=True, help='UUIDs of categories to attach the behavior to. Pass an array of category UUIDs.')
+@click.option("--advanced-settings", "advanced_settings", default=None, help='JSON object overriding the judge\'s online-evaluation configuration. All four fields are required when this is supplied.\n\n**Shape:**\n```\n{\n  "online_evaluation_mode": "continuous" | "on_demand",\n  "online_sampling_rate": <number 0-100>,\n  "online_span_triggers": [\n    {"field":"span_name"|"span_attribute","operator":"contains"|"equals"|"exists","value":"<string>","key":"<attr-key>"?}\n  ],\n  "online_session_scoring": <bool>\n}\n```\n\n`continuous` runs the judge automatically on qualifying spans; `on_demand` requires a manual `judgment traces evaluate` call. `online_sampling_rate` is a percent (0–100) of matching spans to score.')
+@click.option("--judge-id", "judge_id", default=None, help='Attach the new behavior to an existing judge instead of creating one. The judge must be `score_type=binary` and have no existing behaviors.')
 @click.pass_context
 def behaviors_create_binary(ctx, project_id, name, prompt, description, model, category_ids, advanced_settings, judge_id):
-    'Create a binary (Yes/No) behavior backed by a prompt judge.\n\n\x08\nScores each trace/span as "Yes" (true) or "No" (false) using the prompt you supply. Pass --judge-id to attach to an existing judge instead of creating a new prompt scorer. Requires the developer role.\n\n\x08\n\x08\nExample: judgment behaviors create-binary <PROJECT_ID> \'Relevance\' \\\n  \'Was the assistant response relevant to the user message?\' \\\n  --description \'Binary relevance check\' --model gpt-5.2'
+    'Create a binary (yes/no) behavior.\n\n\x08\nCreate a binary behavior. The judge LLM uses your prompt to decide true/false on each qualifying span.'
     url = "/behaviors/create-binary"
     body = {}
     body["project_id"] = project_id
@@ -183,14 +182,14 @@ def behaviors_create_binary(ctx, project_id, name, prompt, description, model, c
 @click.argument("project_id")
 @click.argument("name")
 @click.argument("prompt")
-@click.option("--options", "options", required=True, help='JSON array of the allowed output categories the classifier judge can return. Must contain at least one option.\n\n\x08\nShape:\n  [\n    {"name":"<label>", "description":"<optional human description>", "category_ids":["<uuid>", ...]},\n    ...\n  ]\n\nExample: --options \'[{"name":"Good","description":"helpful response"},{"name":"Bad","description":"unhelpful or incorrect"}]\'')
+@click.option("--options", "options", required=True, help='JSON array of the allowed output categories the classifier judge can return. Must contain at least one option.\n\n**Shape:**\n```\n[\n  {"name":"<label>", "description":"<optional human description>", "category_ids":["<uuid>", ...]},\n  ...\n]\n```')
 @click.option("--model", "model", default=None, help='LLM model ID used by the judge prompt. Defaults to "gpt-5.2" when omitted.')
-@click.option("--category-ids", "category_ids", multiple=True, help='UUID of a category to attach the behavior to. Repeat the flag to attach multiple categories: --category-ids <uuid1> --category-ids <uuid2>.')
-@click.option("--advanced-settings", "advanced_settings", default=None, help='JSON object overriding the judge\'s online-evaluation configuration. All four fields are required when this flag is supplied.\n\n\x08\nShape:\n  {\n    "online_evaluation_mode": "continuous" | "on_demand",\n    "online_sampling_rate": <number 0-100>,\n    "online_span_triggers": [\n      {"field":"span_name"|"span_attribute","operator":"contains"|"equals"|"exists","value":"<string>","key":"<attr-key>"?}\n    ],\n    "online_session_scoring": <bool>\n  }\n\n"continuous" runs the judge automatically on qualifying spans; "on_demand" requires a manual `judgment traces evaluate` call. online_sampling_rate is a percent of matching spans to score.\n\nExample: --advanced-settings \'{"online_evaluation_mode":"continuous","online_sampling_rate":10,"online_span_triggers":[],"online_session_scoring":false}\'')
-@click.option("--judge-id", "judge_id", default=None, help='Attach the new behavior to an existing judge instead of creating one. The judge must be score_type=categorical and have no existing behaviors.')
+@click.option("--category-ids", "category_ids", multiple=True, help='UUIDs of categories to attach the behavior to. Pass an array of category UUIDs.')
+@click.option("--advanced-settings", "advanced_settings", default=None, help='JSON object overriding the judge\'s online-evaluation configuration. All four fields are required when this is supplied.\n\n**Shape:**\n```\n{\n  "online_evaluation_mode": "continuous" | "on_demand",\n  "online_sampling_rate": <number 0-100>,\n  "online_span_triggers": [\n    {"field":"span_name"|"span_attribute","operator":"contains"|"equals"|"exists","value":"<string>","key":"<attr-key>"?}\n  ],\n  "online_session_scoring": <bool>\n}\n```\n\n`continuous` runs the judge automatically on qualifying spans; `on_demand` requires a manual `judgment traces evaluate` call. `online_sampling_rate` is a percent (0–100) of matching spans to score.')
+@click.option("--judge-id", "judge_id", default=None, help='Attach the new behavior to an existing judge instead of creating one. The judge must be `score_type=categorical` and have no existing behaviors.')
 @click.pass_context
 def behaviors_create_classifier(ctx, project_id, name, prompt, options, model, category_ids, advanced_settings, judge_id):
-    'Create a categorical (multi-class) behavior backed by a prompt judge.\n\n\x08\nScores each trace/span into one of the --options categories. Pass --judge-id to attach to an existing classifier judge instead of creating a new one. Requires the developer role.\n\n\x08\n\x08\nExample: judgment behaviors create-classifier <PROJECT_ID> \'Sentiment\' \\\n  \'Classify the assistant response tone.\' \\\n  --options \'[{"name":"Positive"},{"name":"Neutral"},{"name":"Negative"}]\''
+    'Create a classifier (multi-label) behavior.\n\n\x08\nCreate a classifier behavior. The judge LLM picks one of the supplied options for each qualifying span.'
     url = "/behaviors/create-classifier"
     body = {}
     body["project_id"] = project_id
@@ -213,10 +212,10 @@ def behaviors_create_classifier(ctx, project_id, name, prompt, options, model, c
 @click.argument("project_id")
 @click.argument("behavior_id")
 @click.option("--delete-scorer", "delete_scorer", default=None, type=bool, help='When true, also delete the underlying prompt scorer if no other behaviors reference it.')
-@click.option("--delete-all-values", "delete_all_values", default=None, type=bool, help='For classifier behaviors, when true deletes every category row for this judge (not just the provided BEHAVIOR_ID). Ignored for binary behaviors (those always delete both rows).')
+@click.option("--delete-all-values", "delete_all_values", default=None, type=bool, help='For classifier behaviors, when true deletes every category row for this judge (not just the provided behavior_id). Ignored for binary behaviors.')
 @click.pass_context
 def behaviors_delete(ctx, project_id, behavior_id, delete_scorer, delete_all_values):
-    'Delete a behavior. Requires the admin role.\n\n\x08\nBinary behaviors always delete both the true and false rows. For classifier behaviors, pass --delete-all-values true to delete every category row for the judge at once. Pass --delete-scorer true to also delete the underlying prompt scorer when no other behaviors reference it.\n\n\x08\n\x08\nExample: judgment behaviors delete <PROJECT_ID> <BEHAVIOR_ID> --delete-scorer true'
+    """Delete a behavior."""
     url = "/behaviors/delete"
     body = {}
     body["project_id"] = project_id
@@ -232,11 +231,11 @@ def behaviors_delete(ctx, project_id, behavior_id, delete_scorer, delete_all_val
 @behaviors_group.command("get")
 @click.argument("project_id")
 @click.argument("behavior_id")
-@click.option("--start-date", "start_date", default=None)
-@click.option("--end-date", "end_date", default=None)
+@click.option("--start-date", "start_date", default=None, help='Optional ISO 8601 start date for stats.')
+@click.option("--end-date", "end_date", default=None, help='Optional ISO 8601 end date for stats.')
 @click.pass_context
 def behaviors_get(ctx, project_id, behavior_id, start_date, end_date):
-    """Get behavior detail."""
+    """Get a behavior with judge details and stats."""
     url = "/behaviors/detail"
     body = {}
     body["project_id"] = project_id
@@ -253,7 +252,7 @@ def behaviors_get(ctx, project_id, behavior_id, start_date, end_date):
 @click.argument("project_id")
 @click.pass_context
 def behaviors_list(ctx, project_id):
-    """List behaviors."""
+    'List behaviors.\n\n\x08\nList every behavior in a project along with rolled-up trace counts and last-seen stats.'
     url = "/behaviors/list"
     body = {}
     body["project_id"] = project_id
@@ -267,7 +266,7 @@ def behaviors_list(ctx, project_id):
 @click.option("--description", "description", default=None, help='New human-readable description for the behavior.')
 @click.pass_context
 def behaviors_update(ctx, project_id, behavior_id, description):
-    "Update a behavior's description. Requires the developer role.\n\n\x08\n\x08\nExample: judgment behaviors update <PROJECT_ID> <BEHAVIOR_ID> --description 'New description'"
+    """Update a behavior’s description."""
     url = "/behaviors/update"
     body = {}
     body["project_id"] = project_id
@@ -277,6 +276,7 @@ def behaviors_update(ctx, project_id, behavior_id, description):
     result = ctx.obj["client"].request("POST", url, json_body=body)
     _output(result)
 
+
 # ────────────────────────────────────────────────────────────────────
 # Group: docs
 # ────────────────────────────────────────────────────────────────────
@@ -284,15 +284,14 @@ def behaviors_update(ctx, project_id, behavior_id, description):
 
 @click.group("docs")
 def docs_group() -> None:
-    """Search and read docs."""
-    pass
+    'Search and read Judgment documentation.'
 
 
-@docs_group.command("read")
+@docs_group.command("get-page")
 @click.argument("path")
 @click.pass_context
-def docs_read(ctx, path):
-    """Read doc page."""
+def docs_get_page(ctx, path):
+    'Read a documentation page.\n\n\x08\nFetch the rendered text of a documentation page by its path. Use `docs.search` first to find candidate paths.'
     url = "/docs/page"
     params = {}
     params["path"] = path
@@ -302,10 +301,10 @@ def docs_read(ctx, path):
 
 @docs_group.command("search")
 @click.argument("query")
-@click.option("--match-count", "match_count", default=None, type=float)
+@click.option("--match-count", "match_count", default=None, type=float, help='Maximum results to return (1–20). Defaults to 8 when omitted.')
 @click.pass_context
 def docs_search(ctx, query, match_count):
-    """Search docs."""
+    'Search docs.\n\n\x08\nHybrid (vector + lexical) search over the public Judgment documentation. Returns the top matching headings with full URLs.'
     url = "/docs/search"
     body = {}
     body["query"] = query
@@ -314,6 +313,7 @@ def docs_search(ctx, query, match_count):
     result = ctx.obj["client"].request("POST", url, json_body=body)
     _output(result)
 
+
 # ────────────────────────────────────────────────────────────────────
 # Group: judges
 # ────────────────────────────────────────────────────────────────────
@@ -321,16 +321,15 @@ def docs_search(ctx, query, match_count):
 
 @click.group("judges")
 def judges_group() -> None:
-    """Manage judges (scorers)."""
-    pass
+    'View and manage judges and their online-evaluation settings.'
 
 
-@judges_group.command("settings")
+@judges_group.command("get-settings")
 @click.argument("project_id")
 @click.argument("judge_id")
 @click.pass_context
-def judges_settings(ctx, project_id, judge_id):
-    """Get judge settings."""
+def judges_get_settings(ctx, project_id, judge_id):
+    """Get a judge’s online-evaluation settings."""
     url = "/judges/settings"
     body = {}
     body["project_id"] = project_id
@@ -342,13 +341,13 @@ def judges_settings(ctx, project_id, judge_id):
 @judges_group.command("update-settings")
 @click.argument("project_id")
 @click.argument("judge_id")
-@click.argument("evaluation_mode", type=click.Choice(['continuous', 'on_demand']))
-@click.argument("sampling_rate", type=float)
-@click.option("--span-triggers", "span_triggers", default=None, help='JSON array of span filters that restrict which spans the judge evaluates. Pass [] to evaluate all spans.\n\n\x08\nShape:\n  [\n    {\n      "field": "span_name" | "span_attribute",\n      "operator": "contains" | "equals" | "exists",\n      "value": "<string>",\n      "key": "<attribute key>"?\n    },\n    ...\n  ]\n\nUse "field":"span_name" to match on span names; "field":"span_attribute" with "key":"<attr>" to match on a span attribute\'s value. Triggers are ANDed together — a span must match every entry to be evaluated.\n\nExample: --span-triggers \'[{"field":"span_name","operator":"equals","value":"agent.run"}]\'')
+@click.option("--evaluation-mode", "evaluation_mode", required=True, type=click.Choice(['continuous', 'on_demand']))
+@click.option("--sampling-rate", "sampling_rate", required=True, type=float, help='Percent (0–100) of qualifying spans to score.')
+@click.option("--span-triggers", "span_triggers", default=None, help='JSON array of span filters that restrict which spans the judge evaluates. Pass `[]` to evaluate all spans.\n\n**Shape:**\n```\n[\n  {\n    "field": "span_name" | "span_attribute",\n    "operator": "contains" | "equals" | "exists",\n    "value": "<string>",\n    "key": "<attribute key>"?\n  },\n  ...\n]\n```\n\nUse `field: "span_name"` to match on span names; `field: "span_attribute"` with `key: "<attr>"` to match on a span attribute\'s value. Triggers are ANDed together — a span must match every entry to be evaluated.')
 @click.option("--session-scoring", "session_scoring", default=None, type=bool, help='When true, run the judge at session granularity instead of per-span.')
 @click.pass_context
 def judges_update_settings(ctx, project_id, judge_id, evaluation_mode, sampling_rate, span_triggers, session_scoring):
-    'Update a judge\'s online-evaluation configuration.\n\n\x08\nApplies to all behaviors backed by the judge. EVALUATION_MODE selects whether the judge runs continuously on every matching span ("continuous") or only when explicitly invoked ("on_demand"); SAMPLING_RATE is a percentage (0-100) of qualifying spans to score. Requires the developer role.\n\n\x08\n\x08\nExample: judgment judges update-settings <PROJECT_ID> <JUDGE_ID> continuous 10 \\\n  --span-triggers \'[{"field":"span_name","operator":"equals","value":"agent.run"}]\''
+    'Update a judge’s online-evaluation settings.\n\n\x08\nUpdate how often and on which spans a judge runs online. Pass `evaluation_mode: continuous` with a sampling rate to evaluate automatically, or `on_demand` to require manual `judgment traces evaluate` calls.'
     url = "/judges/update-settings"
     body = {}
     body["project_id"] = project_id
@@ -362,6 +361,7 @@ def judges_update_settings(ctx, project_id, judge_id, evaluation_mode, sampling_
     result = ctx.obj["client"].request("POST", url, json_body=body)
     _output(result)
 
+
 # ────────────────────────────────────────────────────────────────────
 # Group: projects
 # ────────────────────────────────────────────────────────────────────
@@ -369,15 +369,14 @@ def judges_update_settings(ctx, project_id, judge_id, evaluation_mode, sampling_
 
 @click.group("projects")
 def projects_group() -> None:
-    """Manage projects."""
-    pass
+    'Manage projects in your organization.'
 
 
 @projects_group.command("add-favorite")
 @click.argument("project_id")
 @click.pass_context
 def projects_add_favorite(ctx, project_id):
-    'Mark a project as a favorite for your user so it appears pinned in the UI.\n\n\x08\n\x08\nExample: judgment projects add-favorite <PROJECT_ID>'
+    'Add project to favorites.\n\n\x08\nMark a project as a favorite for your user so it appears pinned in the UI.'
     url = "/projects/add-favorite"
     body = {}
     body["project_id"] = project_id
@@ -389,7 +388,7 @@ def projects_add_favorite(ctx, project_id):
 @click.argument("project_name")
 @click.pass_context
 def projects_create(ctx, project_name):
-    'Create a new project in your organization. Requires the developer role.\n\n\x08\n\x08\nExample: judgment projects create my-new-project'
+    'Create project.\n\n\x08\nCreate a new project in your organization. Requires the developer role.'
     url = "/projects/create"
     body = {}
     body["project_name"] = project_name
@@ -400,7 +399,7 @@ def projects_create(ctx, project_name):
 @projects_group.command("list")
 @click.pass_context
 def projects_list(ctx):
-    """List projects."""
+    'List projects.\n\n\x08\nList every project in your organization that you have access to.'
     url = "/projects"
     result = ctx.obj["client"].request("GET", url)
     _output(result)
@@ -410,13 +409,14 @@ def projects_list(ctx):
 @click.argument("project_id")
 @click.pass_context
 def projects_remove_favorite(ctx, project_id):
-    "Remove a project from your user's favorites.\n\n\x08\n\x08\nExample: judgment projects remove-favorite <PROJECT_ID>"
+    "Remove project from favorites.\n\n\x08\nRemove a project from your user's favorites."
     url = "/projects/remove-favorite"
     body = {}
     body["project_id"] = project_id
     result = ctx.obj["client"].request("POST", url, json_body=body)
     _output(result)
 
+
 # ────────────────────────────────────────────────────────────────────
 # Group: sessions
 # ────────────────────────────────────────────────────────────────────
@@ -424,8 +424,7 @@ def projects_remove_favorite(ctx, project_id):
 
 @click.group("sessions")
 def sessions_group() -> None:
-    """View and manage sessions."""
-    pass
+    'Search and inspect sessions in a project.'
 
 
 @sessions_group.command("get")
@@ -444,13 +443,13 @@ def sessions_get(ctx, project_id, session_id):
 
 @sessions_group.command("search")
 @click.argument("project_id")
-@click.option("--filters", "filters", default=None, help='Filter expressions, ANDed together. Each item is {"field":<field>,"op":<op>,"value":<value>}. Allowed ops depend on the field\'s type.\n\n\x08\nOp groups:\n  STRING_OPS  = "=" | "!=" | "contains" | "does_not_contain"\n  NUMERIC_OPS = "=" | "!=" | "<" | "<=" | ">" | ">="\n  ARRAY_ANY   = "any"   (matches when the row\'s array overlaps the supplied values)\n\nString fields (op in STRING_OPS, value is a string): session_id.\n\nNumeric fields (op in NUMERIC_OPS, value is a number): trace_count, latency (nanoseconds), total_cost (USD).\n\nArray fields (op = "any", value is an array): behaviors (behavior UUIDs).\n\nExample: --filters \'[{"field":"trace_count","op":">=","value":5}]\'\n')
-@click.option("--time-range", "time_range", default=None, help='{"start_time":<iso8601-string>|null,"end_time":<iso8601-string>|null}. Either bound may be null to leave that side open. Invalid timestamps return 400.')
-@click.option("--pagination", "pagination", required=True, help='{"limit":<int 1-200>,"cursorSortValue":<string>|null,"cursorItemId":<string>|null}.\n\nFirst page: pass null for both cursor fields. Each response returns nextCursor:{sort_value,session_id} (or null when hasMore=false); copy those into cursorSortValue and cursorItemId for the next page.')
-@click.option("--sort-by", "sort_by", default=None, help='{"field":<sort_field>,"direction":"asc"|"desc"} where sort_field is one of: created_at, num_traces, latency, llm_cost. Default when omitted: {"field":"created_at","direction":"desc"}.')
+@click.option("--filters", "filters", default=None, help='Filter expressions, ANDed together. Each item is `{"field":<field>,"op":<op>,"value":<value>}`. Allowed ops depend on the field\'s type.\n\n**Op groups:**\n- `STRING_OPS` = `=` | `!=` | `contains` | `does_not_contain`\n- `NUMERIC_OPS` = `=` | `!=` | `<` | `<=` | `>` | `>=`\n- `ARRAY_ANY` = `any` (matches when the row\'s array overlaps the supplied values)\n\n**String fields** (op in STRING_OPS, value is a string): `session_id`.\n\n**Numeric fields** (op in NUMERIC_OPS, value is a number): `trace_count`, `latency` (nanoseconds), `total_cost` (USD).\n\n**Array fields** (op = `any`, value is an array): `behaviors` (behavior UUIDs).')
+@click.option("--time-range", "time_range", default=None, help='`{"start_time":<iso8601-string>|null,"end_time":<iso8601-string>|null}`. Either bound may be null to leave that side open. Invalid timestamps return 400.')
+@click.option("--pagination", "pagination", required=True, help='`{"limit":<int 1-200>,"cursorSortValue":<string>|null,"cursorItemId":<string>|null}`.\n\nFirst page: pass null for both cursor fields. Each response returns `nextCursor:{sort_value,session_id}` (or null when `hasMore=false`); copy those into `cursorSortValue` and `cursorItemId` for the next page.')
+@click.option("--sort-by", "sort_by", default=None, help='`{"field":<sort_field>,"direction":"asc"|"desc"}` where `sort_field` is one of: `created_at`, `num_traces`, `latency`, `llm_cost`. Default when omitted: `{"field":"created_at","direction":"desc"}`.')
 @click.pass_context
 def sessions_search(ctx, project_id, filters, time_range, pagination, sort_by):
-    'Search sessions in a project.\n\n\x08\nFiltering, sorting, time bounds, and pagination are passed as JSON via the body fields below — see each flag\'s description for its full reference.\n\n\x08\n\x08\nExample: judgment sessions search <PROJECT_ID> \\\n  --filters \'[{"field":"trace_count","op":">=","value":5}]\' \\\n  --sort-by \'{"field":"latency","direction":"desc"}\' \\\n  --pagination \'{"limit":25,"cursorSortValue":null,"cursorItemId":null}\''
+    'Search sessions.\n\n\x08\nFilter, sort, time-bound, and paginate sessions in a project.'
     url = "/sessions/search"
     body = {}
     body["project_id"] = project_id
@@ -470,7 +469,7 @@ def sessions_search(ctx, project_id, filters, time_range, pagination, sort_by):
 @click.argument("session_id")
 @click.pass_context
 def sessions_trace_behaviors(ctx, project_id, session_id):
-    """Get session trace behaviors."""
+    """List behaviors observed across a session’s traces."""
     url = "/sessions/trace-behaviors"
     body = {}
     body["project_id"] = project_id
@@ -484,7 +483,7 @@ def sessions_trace_behaviors(ctx, project_id, session_id):
 @click.argument("session_id")
 @click.pass_context
 def sessions_trace_ids(ctx, project_id, session_id):
-    """Get session trace IDs."""
+    """List trace IDs in a session."""
     url = "/sessions/trace-ids"
     body = {}
     body["project_id"] = project_id
@@ -492,6 +491,7 @@ def sessions_trace_ids(ctx, project_id, session_id):
     result = ctx.obj["client"].request("POST", url, json_body=body)
     _output(result)
 
+
 # ────────────────────────────────────────────────────────────────────
 # Group: traces
 # ────────────────────────────────────────────────────────────────────
@@ -499,17 +499,16 @@ def sessions_trace_ids(ctx, project_id, session_id):
 
 @click.group("traces")
 def traces_group() -> None:
-    """View and manage traces."""
-    pass
+    'Search, inspect, tag, and re-evaluate traces in a project.'
 
 
 @traces_group.command("add-tags")
 @click.argument("project_id")
 @click.argument("trace_id")
-@click.option("--tags", "tags", multiple=True, help='Tag string to attach to the trace. Repeat --tags to attach multiple tags: --tags a --tags b.')
+@click.option("--tags", "tags", multiple=True, required=True, help='String tags to attach to the trace. Tags are additive — existing tags on the trace are preserved.')
 @click.pass_context
 def traces_add_tags(ctx, project_id, trace_id, tags):
-    'Attach one or more string tags to an existing trace.\n\n\x08\nTags are additive — existing tags on the trace are preserved. Repeat --tags to add multiple tags.\n\n\x08\n\x08\nExample: judgment traces add-tags <PROJECT_ID> <TRACE_ID> \\\n  --tags regression --tags investigate'
+    'Add tags to a trace.\n\n\x08\nAttach one or more string tags to an existing trace. Tags are additive — existing tags are preserved.'
     url = "/traces/add-tags"
     body = {}
     body["project_id"] = project_id
@@ -525,7 +524,7 @@ def traces_add_tags(ctx, project_id, trace_id, tags):
 @click.argument("trace_id")
 @click.pass_context
 def traces_behaviors(ctx, project_id, trace_id):
-    """Get trace behaviors."""
+    """List behaviors observed on a trace."""
     url = "/traces/behaviors"
     body = {}
     body["project_id"] = project_id
@@ -536,12 +535,12 @@ def traces_behaviors(ctx, project_id, trace_id):
 
 @traces_group.command("evaluate")
 @click.argument("project_id")
-@click.option("--evaluate-all", "evaluate_all", default=None, type=bool, help='When true, re-evaluate every trace in the project. Mutually exclusive with --trace-ids.')
-@click.option("--trace-ids", "trace_ids", multiple=True, help='Trace UUID to re-evaluate. Repeat --trace-ids to evaluate multiple: --trace-ids <id1> --trace-ids <id2>.')
-@click.option("--specific-judge-names", "specific_judge_names", multiple=True, help='Restrict evaluation to judges with these names. Repeat the flag to pass multiple. Omit to run every applicable judge.')
+@click.option("--evaluate-all", "evaluate_all", default=None, type=bool, help='When true, re-evaluate every trace in the project. Mutually exclusive with `trace_ids`.')
+@click.option("--trace-ids", "trace_ids", multiple=True, help='Trace UUIDs to re-evaluate. Mutually exclusive with `evaluate_all`.')
+@click.option("--specific-judge-names", "specific_judge_names", multiple=True, help='Restrict evaluation to judges with these names. Omit to run every applicable judge.')
 @click.pass_context
 def traces_evaluate(ctx, project_id, evaluate_all, trace_ids, specific_judge_names):
-    "Queue traces for re-evaluation by the project's judges.\n\n\x08\nPass specific trace IDs with --trace-ids (repeat the flag) or use --evaluate-all true to re-evaluate every trace in the project. Use --specific-judge-names to restrict which judges run (repeat the flag). Requires the developer role.\n\n\x08\n\x08\nExample: judgment traces evaluate <PROJECT_ID> \\\n  --trace-ids <TRACE_ID_1> --trace-ids <TRACE_ID_2> \\\n  --specific-judge-names Relevance"
+    'Re-evaluate traces.\n\n\x08\nQueue traces for re-evaluation by the project’s judges. Pass `trace_ids` to re-evaluate specific traces, or `evaluate_all: true` to re-evaluate every trace in the project.'
     url = "/traces/evaluate"
     body = {}
     body["project_id"] = project_id
@@ -560,7 +559,7 @@ def traces_evaluate(ctx, project_id, evaluate_all, trace_ids, specific_judge_nam
 @click.argument("trace_id")
 @click.pass_context
 def traces_get(ctx, project_id, trace_id):
-    """Get trace detail."""
+    """Get a trace by ID."""
     url = "/traces/detail"
     body = {}
     body["project_id"] = project_id
@@ -571,13 +570,13 @@ def traces_get(ctx, project_id, trace_id):
 
 @traces_group.command("search")
 @click.argument("project_id")
-@click.option("--filters", "filters", default=None, help='Filter expressions, ANDed together. Each item is {"field":<field>,"op":<op>,"value":<value>}. Allowed ops depend on the field\'s type.\n\n\x08\nOp groups:\n  STRING_OPS  = "=" | "!=" | "contains" | "does_not_contain" | "exists" | "is_absent"\n  NUMERIC_OPS = "=" | "!=" | "<" | "<=" | ">" | ">="\n  ARRAY_ANY   = "any"   (matches when the row\'s array overlaps the supplied values)\n\nString fields (op in STRING_OPS, value is a string): span_name, customer_id, customer_user_id, session_id, error, dataset_id.\n\nNumeric fields (op in NUMERIC_OPS, value is a number): duration (nanoseconds), llm_cost (USD).\n\nArray fields (op = "any", value is an array): tags (strings), rules_invoked (rule names from this project, strings), behaviors (behavior UUIDs).\n\n\x08\nSpecial:\n  full_text_search       op = "contains", value is a string searched across span attribute text.\n  span_attributes_roots  matches a single span attribute key/value:\n                           {"field":"span_attributes_roots","key":"<attribute-name>","op":<STRING_OPS>,"value":"<string>"}\n\nExample: --filters \'[{"field":"span_name","op":"=","value":"agent.run"}]\'\n')
-@click.option("--sort-by", "sort_by", default=None, help='{"field":<sort_field>,"direction":"asc"|"desc"} where sort_field is one of: created_at, span_name, duration, llm_cost. Default when omitted: {"field":"created_at","direction":"desc"}.')
-@click.option("--time-range", "time_range", default=None, help='{"start_time":<iso8601-string>|null,"end_time":<iso8601-string>|null}. Either bound may be null to leave that side open. Invalid timestamps return 400.')
-@click.option("--pagination", "pagination", required=True, help='{"limit":<int 1-200>,"cursorSortValue":<string>|null,"cursorItemId":<string>|null}.\n\nFirst page: pass null for both cursor fields. Each response returns nextCursor:{sort_value,trace_id} (or null when hasMore=false); copy those into cursorSortValue and cursorItemId for the next page.')
+@click.option("--filters", "filters", default=None, help='Filter expressions, ANDed together. Each item is `{"field":<field>,"op":<op>,"value":<value>}`. Allowed ops depend on the field\'s type.\n\n**Op groups:**\n- `STRING_OPS` = `=` | `!=` | `contains` | `does_not_contain` | `exists` | `is_absent`\n- `NUMERIC_OPS` = `=` | `!=` | `<` | `<=` | `>` | `>=`\n- `ARRAY_ANY` = `any` (matches when the row\'s array overlaps the supplied values)\n\n**String fields** (op in STRING_OPS, value is a string): `span_name`, `customer_id`, `customer_user_id`, `session_id`, `error`, `dataset_id`.\n\n**Numeric fields** (op in NUMERIC_OPS, value is a number): `duration` (nanoseconds), `llm_cost` (USD).\n\n**Array fields** (op = `any`, value is an array): `tags` (strings), `rules_invoked` (rule names from this project, strings), `behaviors` (behavior UUIDs).\n\n**Special:**\n- `full_text_search`: op = `contains`, value is a string searched across span attribute text.\n- `span_attributes_roots`: matches a single span attribute key/value: `{"field":"span_attributes_roots","key":"<attribute-name>","op":<STRING_OPS>,"value":"<string>"}`')
+@click.option("--sort-by", "sort_by", default=None, help='`{"field":<sort_field>,"direction":"asc"|"desc"}` where `sort_field` is one of: `created_at`, `span_name`, `duration`, `llm_cost`. Default when omitted: `{"field":"created_at","direction":"desc"}`.')
+@click.option("--time-range", "time_range", default=None, help='`{"start_time":<iso8601-string>|null,"end_time":<iso8601-string>|null}`. Either bound may be null to leave that side open. Invalid timestamps return 400.')
+@click.option("--pagination", "pagination", required=True, help='`{"limit":<int 1-200>,"cursorSortValue":<string>|null,"cursorItemId":<string>|null}`.\n\nFirst page: pass null for both cursor fields. Each response returns `nextCursor:{sort_value,trace_id}` (or null when `hasMore=false`); copy those into `cursorSortValue` and `cursorItemId` for the next page.')
 @click.pass_context
 def traces_search(ctx, project_id, filters, sort_by, time_range, pagination):
-    'Search traces in a project.\n\n\x08\nFiltering, sorting, time bounds, and pagination are passed as JSON via the body fields below — see each flag\'s description for its full reference.\n\n\x08\n\x08\nExample: judgment traces search <PROJECT_ID> \\\n  --filters \'[{"field":"span_name","op":"=","value":"agent.run"}]\' \\\n  --sort-by \'{"field":"llm_cost","direction":"desc"}\' \\\n  --pagination \'{"limit":25,"cursorSortValue":null,"cursorItemId":null}\''
+    'Search traces.\n\n\x08\nFilter, sort, time-bound, and paginate traces in a project. See each body field for the exact JSON shape it expects.'
     url = "/traces/search"
     body = {}
     body["project_id"] = project_id
@@ -594,10 +593,10 @@ def traces_search(ctx, project_id, filters, sort_by, time_range, pagination):
 
 @traces_group.command("span")
 @click.argument("project_id")
-@click.option("--spans", "spans", required=True, help='JSON value for spans.')
+@click.option("--spans", "spans", required=True, help='Up to 20 trace/span ID pairs to fetch span details for in a single request.')
 @click.pass_context
 def traces_span(ctx, project_id, spans):
-    """Get trace span details."""
+    'Get span details.\n\n\x08\nFetch full details (inputs/outputs/attributes) for up to 20 specific spans across one or more traces.'
     url = "/traces/span"
     body = {}
     body["project_id"] = project_id
@@ -611,7 +610,7 @@ def traces_span(ctx, project_id, spans):
 @click.argument("trace_id")
 @click.pass_context
 def traces_spans(ctx, project_id, trace_id):
-    """Get trace spans."""
+    """List a trace’s spans."""
     url = "/traces/spans"
     body = {}
     body["project_id"] = project_id
@@ -625,7 +624,7 @@ def traces_spans(ctx, project_id, trace_id):
 @click.argument("trace_id")
 @click.pass_context
 def traces_tags(ctx, project_id, trace_id):
-    """Get trace tags."""
+    """List a trace’s tags."""
     url = "/traces/tags"
     body = {}
     body["project_id"] = project_id
diff --git a/src/judgment_cli/judges.py b/src/judgment_cli/judges.py
index d95ea48..30b2b9d 100644
--- a/src/judgment_cli/judges.py
+++ b/src/judgment_cli/judges.py
@@ -1,9 +1,20 @@
 """Hand-written ``judgment judges`` commands.
 
-The ``judges`` group itself is auto-generated from the OpenAPI spec; the
-commands here (``upload``, ``init``) are attached to that group via
-``attach_to`` because they need behaviour the generator cannot express
-(multipart upload, local file scaffolding).
+The ``judges`` group itself is auto-generated from the OpenAPI spec. The
+commands defined here (``upload``, ``init``) cover behaviour the OpenAPI
+spec cannot express on its own:
+
+* ``upload`` — packages local Python source into a tar+gzip bundle and
+  posts it as multipart form data. The matching server route exists
+  (``judges.upload``) but is intentionally listed in
+  :data:`scripts.generate_cli.MANUAL_COMMANDS` so this implementation owns
+  the slot.
+* ``init`` — pure local scaffolder that writes a starter judge file. There
+  is no server endpoint for this command.
+
+These commands are registered onto the auto-generated ``judges`` group
+through :data:`MANUAL_GROUP_COMMANDS`, which ``main.py`` consumes after
+calling ``register_commands``.
 """
 
 from __future__ import annotations
@@ -276,7 +287,6 @@ def judges_init(
     success(f"Wrote {os.path.abspath(judge_path)}")
 
 
-def attach_to(judges_group: click.Group) -> None:
-    """Attach the hand-written judges commands to the auto-generated group."""
-    judges_group.add_command(judges_upload)
-    judges_group.add_command(judges_init)
+MANUAL_GROUP_COMMANDS: dict[str, list[click.Command]] = {
+    "judges": [judges_upload, judges_init],
+}
diff --git a/src/judgment_cli/main.py b/src/judgment_cli/main.py
index 18bd97c..a4856aa 100644
--- a/src/judgment_cli/main.py
+++ b/src/judgment_cli/main.py
@@ -145,7 +145,26 @@ def status() -> None:
 
 
 register_commands(cli)
-judges.attach_to(cli.commands["judges"])
+
+
+def _attach_manual_commands(root: click.Group, manual: dict[str, list[click.Command]]) -> None:
+    """Attach hand-written commands to their auto-generated parent groups.
+
+    Raises a clear error if a parent group went missing (e.g. the API spec
+    no longer advertises it), instead of silently dropping the command.
+    """
+    for group_name, commands in manual.items():
+        parent = root.commands.get(group_name)
+        if not isinstance(parent, click.Group):
+            raise RuntimeError(
+                f"Manual commands target group '{group_name}' but it is not "
+                "present in the auto-generated CLI. Did the OpenAPI spec change?"
+            )
+        for command in commands:
+            parent.add_command(command)
+
+
+_attach_manual_commands(cli, judges.MANUAL_GROUP_COMMANDS)
 
 
 def main() -> None:

From c38bbf7b3f1495874b2f9a0882b0f0a619ebd1a1 Mon Sep 17 00:00:00 2001
From: Aaryan Divate <divateaaryan@gmail.com>
Date: Fri, 24 Apr 2026 16:10:11 -0700
Subject: [PATCH 5/6] Rename judgment upload command

---
 src/judgment_cli/scorer_bundle.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/judgment_cli/scorer_bundle.py b/src/judgment_cli/scorer_bundle.py
index cc2d1a9..cbec7db 100644
--- a/src/judgment_cli/scorer_bundle.py
+++ b/src/judgment_cli/scorer_bundle.py
@@ -1,6 +1,6 @@
 """Build scorer upload bundles and detect scorer metadata from source.
 
-Used by ``judgment scorer upload`` to package an entrypoint Python file (plus
+Used by ``judgment judges upload`` to package an entrypoint Python file (plus
 optional requirements.txt and additional include paths) into a tar.gz, and to
 infer the scorer's class name and response type by parsing its AST.
 """

From 5cd1b9ef03cb5416c2480e28f782ba8cd20c0076 Mon Sep 17 00:00:00 2001
From: Aaryan Divate <divateaaryan@gmail.com>
Date: Fri, 24 Apr 2026 16:38:17 -0700
Subject: [PATCH 6/6] Remove attach logic, add judges get command

---
 src/judgment_cli/generated_commands.py | 12 ++++++++++++
 src/judgment_cli/judges.py             | 16 ++++++----------
 src/judgment_cli/main.py               | 24 ++++--------------------
 3 files changed, 22 insertions(+), 30 deletions(-)

diff --git a/src/judgment_cli/generated_commands.py b/src/judgment_cli/generated_commands.py
index 02dadfa..0537199 100644
--- a/src/judgment_cli/generated_commands.py
+++ b/src/judgment_cli/generated_commands.py
@@ -338,6 +338,18 @@ def judges_get_settings(ctx, project_id, judge_id):
     _output(result)
 
 
+@judges_group.command("list")
+@click.argument("project_id")
+@click.pass_context
+def judges_list(ctx, project_id):
+    'List judges in a project.\n\n\x08\nList every judge in a project, including prompt, code, and custom (uploaded) judges. Returns each judge with its current configuration and online-evaluation settings.'
+    url = "/judges/list"
+    body = {}
+    body["project_id"] = project_id
+    result = ctx.obj["client"].request("POST", url, json_body=body)
+    _output(result)
+
+
 @judges_group.command("update-settings")
 @click.argument("project_id")
 @click.argument("judge_id")
diff --git a/src/judgment_cli/judges.py b/src/judgment_cli/judges.py
index 30b2b9d..ba9bb8e 100644
--- a/src/judgment_cli/judges.py
+++ b/src/judgment_cli/judges.py
@@ -12,9 +12,9 @@
 * ``init`` — pure local scaffolder that writes a starter judge file. There
   is no server endpoint for this command.
 
-These commands are registered onto the auto-generated ``judges`` group
-through :data:`MANUAL_GROUP_COMMANDS`, which ``main.py`` consumes after
-calling ``register_commands``.
+Commands are attached directly to the auto-generated ``judges_group`` via
+the standard ``@judges_group.command(...)`` decorator — importing this
+module is enough to register them.
 """
 
 from __future__ import annotations
@@ -28,10 +28,11 @@
 
 from judgment_cli import scorer_bundle
 from judgment_cli.client import JudgmentClient
+from judgment_cli.generated_commands import judges_group
 from judgment_cli.ui import error, output, success
 
 
-@click.command("upload")
+@judges_group.command("upload")
 @click.argument("entrypoint_path", type=click.Path(exists=True, dir_okay=False))
 @click.option(
     "-p",
@@ -213,7 +214,7 @@ async def score(self, data: Example) -> {name}Response:
 }
 
 
-@click.command("init")
+@judges_group.command("init")
 @click.option(
     "-t",
     "--response-type",
@@ -285,8 +286,3 @@ def judges_init(
         )
     judge_path.write_text(_TEMPLATES[response_type].format(name=judge_name))
     success(f"Wrote {os.path.abspath(judge_path)}")
-
-
-MANUAL_GROUP_COMMANDS: dict[str, list[click.Command]] = {
-    "judges": [judges_upload, judges_init],
-}
diff --git a/src/judgment_cli/main.py b/src/judgment_cli/main.py
index a4856aa..268bc88 100644
--- a/src/judgment_cli/main.py
+++ b/src/judgment_cli/main.py
@@ -6,7 +6,7 @@
 
 from judgment_cli import __version__
 from judgment_cli.client import JudgmentClient
-from judgment_cli import config, judges
+from judgment_cli import config
 from judgment_cli.generated_commands import register_commands
 from judgment_cli.ui import mask_key
 
@@ -146,25 +146,9 @@ def status() -> None:
 
 register_commands(cli)
 
-
-def _attach_manual_commands(root: click.Group, manual: dict[str, list[click.Command]]) -> None:
-    """Attach hand-written commands to their auto-generated parent groups.
-
-    Raises a clear error if a parent group went missing (e.g. the API spec
-    no longer advertises it), instead of silently dropping the command.
-    """
-    for group_name, commands in manual.items():
-        parent = root.commands.get(group_name)
-        if not isinstance(parent, click.Group):
-            raise RuntimeError(
-                f"Manual commands target group '{group_name}' but it is not "
-                "present in the auto-generated CLI. Did the OpenAPI spec change?"
-            )
-        for command in commands:
-            parent.add_command(command)
-
-
-_attach_manual_commands(cli, judges.MANUAL_GROUP_COMMANDS)
+# Hand-written commands attach themselves to the auto-generated groups when
+# imported (see judgment_cli/judges.py). Importing for side effects only.
+from judgment_cli import judges  # noqa: E402, F401
 
 
 def main() -> None: