From 98b0cc94313fe67c425fe791f7c7155f88580fe0 Mon Sep 17 00:00:00 2001 From: Radu Raicea Date: Thu, 30 Jul 2026 09:32:54 -0400 Subject: [PATCH 1/4] feat(inbox): make AI observability source available to everyone Remove the staff-only "Internal" badge and tooltip from the AI observability card in the Self-driving source config, so it's presented as generally available rather than internal. Generated-By: PostHog Code Task-Id: 9725e5be-8b91-44f8-8c12-3419a5bf080c --- .../src/features/inbox/components/ResponderAgentRoster.tsx | 7 +------ .../src/features/inbox/components/SignalSourceToggles.tsx | 5 +---- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/packages/ui/src/features/inbox/components/ResponderAgentRoster.tsx b/packages/ui/src/features/inbox/components/ResponderAgentRoster.tsx index 05ad62bbc0..627fd65c44 100644 --- a/packages/ui/src/features/inbox/components/ResponderAgentRoster.tsx +++ b/packages/ui/src/features/inbox/components/ResponderAgentRoster.tsx @@ -14,7 +14,7 @@ import type { SignalSourceValues } from "@posthog/ui/features/inbox/components/S import { InboxBadge } from "@posthog/ui/features/inbox/components/utils/InboxBadge"; import { getSourceProductMeta } from "@posthog/ui/features/inbox/components/utils/source-product-icons"; import { Badge } from "@posthog/ui/primitives/Badge"; -import { Box, Flex, Spinner, Switch, Text, Tooltip } from "@radix-ui/themes"; +import { Box, Flex, Spinner, Switch, Text } from "@radix-ui/themes"; import { type ComponentType, memo, useCallback } from "react"; type AgentRosterStatus = "standby" | "watching" | "syncing" | "sync_failed"; @@ -289,11 +289,6 @@ const EvaluationsAgentCard = memo(function EvaluationsAgentCard({ AI Observability - - - Internal - - Quality problems in your AI features. diff --git a/packages/ui/src/features/inbox/components/SignalSourceToggles.tsx b/packages/ui/src/features/inbox/components/SignalSourceToggles.tsx index a7d9816c3a..13f735cb64 100644 --- a/packages/ui/src/features/inbox/components/SignalSourceToggles.tsx +++ b/packages/ui/src/features/inbox/components/SignalSourceToggles.tsx @@ -16,7 +16,7 @@ import { } from "@posthog/shared"; import { getSourceProductMeta } from "@posthog/ui/features/inbox/components/utils/source-product-icons"; import { Badge } from "@posthog/ui/primitives/Badge"; -import { Box, Flex, Spinner, Switch, Text, Tooltip } from "@radix-ui/themes"; +import { Box, Flex, Spinner, Switch, Text } from "@radix-ui/themes"; import { memo, useCallback } from "react"; export type SignalSourceValues = Record; @@ -233,9 +233,6 @@ export const EvaluationsSection = memo(function EvaluationsSection({ AI observability - - Internal - Monitor how your AI features are performing From 010f37526d8059f790740971af6cf95c82f8a676 Mon Sep 17 00:00:00 2001 From: Radu Raicea Date: Thu, 30 Jul 2026 09:54:34 -0400 Subject: [PATCH 2/4] feat(inbox): make AI observability a real source toggle Replace the AI observability "Open" link-out card with an on/off toggle like the other sources, matching the web app's AI observability source card. The toggle governs the project-level `evaluation_report` signal source. `llm_analytics` also carries per-evaluation `evaluation` configs (managed on the evaluations page); the toggle is scoped to `evaluation_report` so those are left untouched. Wired in both the responder roster (Inbox 2.0) and the settings source toggles, plus the parallel core SignalSourceService, with tests covering the scoping. Generated-By: PostHog Code Task-Id: 9725e5be-8b91-44f8-8c12-3419a5bf080c --- .../src/inbox/signalSourceService.test.ts | 41 +++++++++ .../core/src/inbox/signalSourceService.ts | 25 +++++- packages/shared/src/inbox-types.ts | 10 +-- .../components/ConfigureAgentsSection.tsx | 2 - .../inbox/components/ResponderAgentRoster.tsx | 74 ---------------- .../inbox/components/SignalSourceToggles.tsx | 85 +++---------------- .../inbox/components/responderAgentMeta.ts | 8 ++ .../inbox/hooks/useSignalSourceToggles.ts | 26 +++++- 8 files changed, 114 insertions(+), 157 deletions(-) diff --git a/packages/core/src/inbox/signalSourceService.test.ts b/packages/core/src/inbox/signalSourceService.test.ts index 1d7c0c9d87..a835ec7a41 100644 --- a/packages/core/src/inbox/signalSourceService.test.ts +++ b/packages/core/src/inbox/signalSourceService.test.ts @@ -67,6 +67,20 @@ describe("computeSourceValues", () => { ]); expect(values.health_checks).toBe(true); }); + + it("scopes llm_analytics to evaluation_report, ignoring per-evaluation evaluation configs", () => { + // A per-evaluation `evaluation` config must not light up the source-level toggle. + const onlyEvaluation = computeSourceValues([ + config("llm_analytics", "evaluation", true), + ]); + expect(onlyEvaluation.llm_analytics).toBe(false); + + const withReport = computeSourceValues([ + config("llm_analytics", "evaluation", true), + config("llm_analytics", "evaluation_report", true), + ]); + expect(withReport.llm_analytics).toBe(true); + }); }); describe("deriveSourceStates", () => { @@ -124,6 +138,33 @@ describe("SignalSourceService.toggleSource", () => { }); }); + it("creates an llm_analytics config with the evaluation_report source type", async () => { + const client = fakeClient(); + const service = new SignalSourceService(); + await service.toggleSource(client, 1, "llm_analytics", true, [], []); + expect(client.createSignalSourceConfig).toHaveBeenCalledWith(1, { + source_product: "llm_analytics", + source_type: "evaluation_report", + enabled: true, + }); + }); + + it("toggles the evaluation_report config off without touching a co-existing evaluation config", async () => { + const client = fakeClient(); + const service = new SignalSourceService(); + const configs = [ + config("llm_analytics", "evaluation", true), + config("llm_analytics", "evaluation_report", true), + ]; + await service.toggleSource(client, 1, "llm_analytics", false, configs, []); + expect(client.updateSignalSourceConfig).toHaveBeenCalledTimes(1); + expect(client.updateSignalSourceConfig).toHaveBeenCalledWith( + 1, + "llm_analytics-evaluation_report", + { enabled: false }, + ); + }); + it("ensures the issues table syncs with full_refresh for github before enabling", async () => { const client = fakeClient(); const service = new SignalSourceService(); diff --git a/packages/core/src/inbox/signalSourceService.ts b/packages/core/src/inbox/signalSourceService.ts index 92fb48ce63..34a57bb782 100644 --- a/packages/core/src/inbox/signalSourceService.ts +++ b/packages/core/src/inbox/signalSourceService.ts @@ -37,6 +37,10 @@ const SOURCE_TYPE_MAP: Partial> = { conversations: "ticket", health_checks: "health_issue", session_replay: "session_analysis_cluster", + // `llm_analytics` also emits per-evaluation `evaluation` signals configured on the + // evaluations page; this toggle only governs the project-level `evaluation_report` source, + // matching the web app's AI observability source card. + llm_analytics: "evaluation_report", ...Object.fromEntries( EXTERNAL_INBOX_SOURCES.map((s) => [s.product, s.recordKind]), ), @@ -67,9 +71,17 @@ const ALL_SOURCE_PRODUCTS: SignalSourceProduct[] = [ "error_tracking", "health_checks", "session_replay", + "llm_analytics", ...EXTERNAL_INBOX_SOURCES.map((s) => s.product), ]; +// Products whose enabled state must match a single, specific `source_type` rather than any +// config under the product — `llm_analytics` also carries per-evaluation `evaluation` configs +// that this toggle must ignore. +const TYPE_SCOPED_SOURCES: Partial> = { + llm_analytics: "evaluation_report", +}; + function isWarehouseSource(product: SignalSourceProduct): boolean { return product in DATA_WAREHOUSE_SOURCES; } @@ -110,8 +122,12 @@ export function computeSourceValues( ), ); } else { + const scopedType = TYPE_SCOPED_SOURCES[product]; result[product] = configs.some( - (c) => c.source_product === product && c.enabled, + (c) => + c.source_product === product && + (!scopedType || c.source_type === scopedType) && + c.enabled, ); } } @@ -296,8 +312,13 @@ export class SignalSourceService { enabled: boolean, configs: SignalSourceConfig[] | undefined, ): Promise { - const existing = configs?.find((c) => c.source_product === product); const sourceType = SOURCE_TYPE_MAP[product]; + const scopedType = TYPE_SCOPED_SOURCES[product]; + const existing = configs?.find( + (c) => + c.source_product === product && + (!scopedType || c.source_type === scopedType), + ); if (existing) { await client.updateSignalSourceConfig(projectId, existing.id, { enabled, diff --git a/packages/shared/src/inbox-types.ts b/packages/shared/src/inbox-types.ts index 89501ced1c..58b3f7c547 100644 --- a/packages/shared/src/inbox-types.ts +++ b/packages/shared/src/inbox-types.ts @@ -424,13 +424,10 @@ export type SourceProduct = | ExternalInboxSourceProduct; /** - * Products that render as a toggle in the Self-driving sources modal. Excludes non-toggle - * products (`llm_analytics`, `signals_scout`) that appear only as signal origins. + * Products that render as a toggle in the Self-driving sources modal. Excludes `signals_scout`, + * which appears only as a signal origin and is always on rather than user-toggled. */ -export type ToggleableSourceProduct = Exclude< - SourceProduct, - "llm_analytics" | "signals_scout" ->; +export type ToggleableSourceProduct = Exclude; /** * Every backend signal `source_type`: the PostHog-native types (alphabetical) plus the @@ -439,6 +436,7 @@ export type ToggleableSourceProduct = Exclude< export type SourceType = | "cross_source_issue" | "evaluation" + | "evaluation_report" | "health_issue" | "issue_created" | "issue_reopened" diff --git a/packages/ui/src/features/inbox/components/ConfigureAgentsSection.tsx b/packages/ui/src/features/inbox/components/ConfigureAgentsSection.tsx index 3f98471ae2..5dc82aab4d 100644 --- a/packages/ui/src/features/inbox/components/ConfigureAgentsSection.tsx +++ b/packages/ui/src/features/inbox/components/ConfigureAgentsSection.tsx @@ -84,7 +84,6 @@ export function ConfigureAgentsSection() { handleSetupCancel, userAutonomyConfig, userAutonomyConfigLoading, - evaluationsUrl, } = useSignalSourceManager(); const { hasGithubIntegration, isLoadingIntegrations } = useRepositoryIntegration(); @@ -188,7 +187,6 @@ export function ConfigureAgentsSection() { disabled={!hasGithubIntegration} sourceStates={sourceStates} onSetup={handleSetup} - evaluationsUrl={evaluationsUrl} /> )} diff --git a/packages/ui/src/features/inbox/components/ResponderAgentRoster.tsx b/packages/ui/src/features/inbox/components/ResponderAgentRoster.tsx index 627fd65c44..a93c745f62 100644 --- a/packages/ui/src/features/inbox/components/ResponderAgentRoster.tsx +++ b/packages/ui/src/features/inbox/components/ResponderAgentRoster.tsx @@ -61,7 +61,6 @@ interface ResponderAgentRosterProps { > >; onSetup?: (source: ResponderAgentSource) => void; - evaluationsUrl?: string; } export function ResponderAgentRoster({ @@ -70,7 +69,6 @@ export function ResponderAgentRoster({ disabled, sourceStates, onSetup, - evaluationsUrl, }: ResponderAgentRosterProps) { return ( @@ -91,9 +89,6 @@ export function ResponderAgentRoster({ onSetup={onSetup} /> ))} - {group.label === "PostHog data" && evaluationsUrl ? ( - - ) : null} ))} @@ -264,75 +259,6 @@ function AgentIcon({ ); } -const EvaluationsAgentCard = memo(function EvaluationsAgentCard({ - evaluationsUrl, -}: { - evaluationsUrl: string; -}) { - return ( - window.open(evaluationsUrl, "_blank", "noopener")} - className={[ - AGENT_CARD_BASE_CLASS, - "border-border", - AGENT_CARD_INTERACTIVE_IDLE_CLASS, - ].join(" ")} - > - - - - - - - AI Observability - - - - Quality problems in your AI features. - - - { - e.stopPropagation(); - e.preventDefault(); - window.open( - "https://posthog.com/docs/ai-observability", - "_blank", - "noopener", - ); - }} - className="inline-flex items-center gap-1 text-(--accent-11) no-underline" - > - Learn about AI observability - - - - - - - - - ); -}); - function ResponderAgentCardSkeleton() { return ( diff --git a/packages/ui/src/features/inbox/components/SignalSourceToggles.tsx b/packages/ui/src/features/inbox/components/SignalSourceToggles.tsx index 13f735cb64..924163c6f6 100644 --- a/packages/ui/src/features/inbox/components/SignalSourceToggles.tsx +++ b/packages/ui/src/features/inbox/components/SignalSourceToggles.tsx @@ -210,72 +210,6 @@ const ExternalSourceCard = memo(function ExternalSourceCard({ ); }); -interface EvaluationsSectionProps { - evaluationsUrl: string; -} - -export const EvaluationsSection = memo(function EvaluationsSection({ - evaluationsUrl, -}: EvaluationsSectionProps) { - return ( - window.open(evaluationsUrl, "_blank", "noopener")} - className="cursor-pointer rounded-(--radius-3) border border-border bg-(--color-panel-solid) transition duration-150 hover:border-(--gray-6) hover:bg-(--gray-2) hover:shadow-sm" - > - - - - - - - - - AI observability - - - - Monitor how your AI features are performing - - - { - e.stopPropagation(); - e.preventDefault(); - window.open( - "https://posthog.com/docs/ai-observability", - "_blank", - "noopener", - ); - }} - className="inline-flex items-center gap-[4px] text-(--accent-11) no-underline" - > - Learn about AI observability - - - - - - - - - ); -}); - function SourceRunningIndicator({ status, message, @@ -300,7 +234,6 @@ interface SignalSourceTogglesProps { disabled?: boolean; sourceStates?: Partial>; onSetup?: (source: ToggleableSourceProduct) => void; - evaluationsUrl?: string; } export function SignalSourceToggles({ @@ -309,7 +242,6 @@ export function SignalSourceToggles({ disabled, sourceStates, onSetup, - evaluationsUrl, }: SignalSourceTogglesProps) { const toggleSessionReplay = useCallback( (checked: boolean) => onToggle("session_replay", checked), @@ -327,6 +259,10 @@ export function SignalSourceToggles({ (checked: boolean) => onToggle("health_checks", checked), [onToggle], ); + const toggleLlmAnalytics = useCallback( + (checked: boolean) => onToggle("llm_analytics", checked), + [onToggle], + ); return ( @@ -387,9 +323,16 @@ export function SignalSourceToggles({ ) : undefined } /> - {evaluationsUrl && ( - - )} + } + label="AI observability" + description="Quality problems in your AI features" + checked={value.llm_analytics} + onCheckedChange={toggleLlmAnalytics} + disabled={disabled} + docsUrl="https://posthog.com/docs/ai-observability" + docsLabel="AI observability" + /> diff --git a/packages/ui/src/features/inbox/components/responderAgentMeta.ts b/packages/ui/src/features/inbox/components/responderAgentMeta.ts index 76844498f8..51a6af9a3f 100644 --- a/packages/ui/src/features/inbox/components/responderAgentMeta.ts +++ b/packages/ui/src/features/inbox/components/responderAgentMeta.ts @@ -56,6 +56,14 @@ export const RESPONDER_AGENT_GROUPS: ResponderAgentGroup[] = [ docsLabel: "Session Replay", alpha: true, }, + { + source: "llm_analytics", + sourceProduct: "llm_analytics", + label: "AI observability", + description: "Quality problems in your AI features.", + docsUrl: "https://posthog.com/docs/ai-observability", + docsLabel: "AI observability", + }, ], }, { diff --git a/packages/ui/src/features/inbox/hooks/useSignalSourceToggles.ts b/packages/ui/src/features/inbox/hooks/useSignalSourceToggles.ts index f47c0d2177..d35640d451 100644 --- a/packages/ui/src/features/inbox/hooks/useSignalSourceToggles.ts +++ b/packages/ui/src/features/inbox/hooks/useSignalSourceToggles.ts @@ -24,6 +24,10 @@ const SOURCE_TYPE_MAP: Partial> = { conversations: "ticket", health_checks: "health_issue", session_replay: "session_analysis_cluster", + // `llm_analytics` also emits per-evaluation `evaluation` signals configured on the + // evaluations page; this toggle only governs the project-level `evaluation_report` source, + // matching the web app's AI observability source card. + llm_analytics: "evaluation_report", ...Object.fromEntries( EXTERNAL_INBOX_SOURCES.map((s) => [s.product, s.recordKind]), ), @@ -40,6 +44,7 @@ const SOURCE_LABELS: Partial> = { error_tracking: "Error tracking", health_checks: "Health checks", session_replay: "Session replay", + llm_analytics: "AI observability", ...Object.fromEntries( EXTERNAL_INBOX_SOURCES.map((s) => [s.product, s.label]), ), @@ -64,9 +69,17 @@ const ALL_SOURCE_PRODUCTS: SourceKey[] = [ "error_tracking", "health_checks", "session_replay", + "llm_analytics", ...EXTERNAL_INBOX_SOURCES.map((s) => s.product), ]; +// Products whose enabled state must match a single, specific `source_type` rather than any +// config under the product — `llm_analytics` also carries per-evaluation `evaluation` configs +// that this toggle must ignore. +const TYPE_SCOPED_SOURCES: Partial> = { + llm_analytics: "evaluation_report", +}; + function isSetupSourceProduct(product: SourceKey): boolean { return product in DATA_WAREHOUSE_SOURCES; } @@ -89,8 +102,12 @@ function computeValues( ), ); } else { + const scopedType = TYPE_SCOPED_SOURCES[product]; result[product] = configs.some( - (c) => c.source_product === product && c.enabled, + (c) => + c.source_product === product && + (!scopedType || c.source_type === scopedType) && + c.enabled, ); } } @@ -294,8 +311,13 @@ export function useSignalSourceToggles() { } } } else { - const existing = configs?.find((c) => c.source_product === product); const sourceType = SOURCE_TYPE_MAP[product]; + const scopedType = TYPE_SCOPED_SOURCES[product]; + const existing = configs?.find( + (c) => + c.source_product === product && + (!scopedType || c.source_type === scopedType), + ); if (existing) { await client.updateSignalSourceConfig(projectId, existing.id, { enabled, From b10a54fcd58314c10e23f9b4a7e536b740bab468 Mon Sep 17 00:00:00 2001 From: Radu Raicea Date: Thu, 30 Jul 2026 10:00:20 -0400 Subject: [PATCH 3/4] refactor(inbox): drop unnecessary evaluation_report scoping MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit AI observability signals are always emitted per evaluation report — there's no per-evaluation `evaluation` signal source to guard against — so treat llm_analytics as a plain single-type source like the others. Removes the type-scoping helper and the now-moot scoping tests, keeping the source-type mapping to `evaluation_report`. Generated-By: PostHog Code Task-Id: 9725e5be-8b91-44f8-8c12-3419a5bf080c --- .../src/inbox/signalSourceService.test.ts | 30 ------------------- .../core/src/inbox/signalSourceService.ts | 24 ++------------- .../inbox/hooks/useSignalSourceToggles.ts | 24 ++------------- 3 files changed, 6 insertions(+), 72 deletions(-) diff --git a/packages/core/src/inbox/signalSourceService.test.ts b/packages/core/src/inbox/signalSourceService.test.ts index a835ec7a41..7bbbad130f 100644 --- a/packages/core/src/inbox/signalSourceService.test.ts +++ b/packages/core/src/inbox/signalSourceService.test.ts @@ -67,20 +67,6 @@ describe("computeSourceValues", () => { ]); expect(values.health_checks).toBe(true); }); - - it("scopes llm_analytics to evaluation_report, ignoring per-evaluation evaluation configs", () => { - // A per-evaluation `evaluation` config must not light up the source-level toggle. - const onlyEvaluation = computeSourceValues([ - config("llm_analytics", "evaluation", true), - ]); - expect(onlyEvaluation.llm_analytics).toBe(false); - - const withReport = computeSourceValues([ - config("llm_analytics", "evaluation", true), - config("llm_analytics", "evaluation_report", true), - ]); - expect(withReport.llm_analytics).toBe(true); - }); }); describe("deriveSourceStates", () => { @@ -149,22 +135,6 @@ describe("SignalSourceService.toggleSource", () => { }); }); - it("toggles the evaluation_report config off without touching a co-existing evaluation config", async () => { - const client = fakeClient(); - const service = new SignalSourceService(); - const configs = [ - config("llm_analytics", "evaluation", true), - config("llm_analytics", "evaluation_report", true), - ]; - await service.toggleSource(client, 1, "llm_analytics", false, configs, []); - expect(client.updateSignalSourceConfig).toHaveBeenCalledTimes(1); - expect(client.updateSignalSourceConfig).toHaveBeenCalledWith( - 1, - "llm_analytics-evaluation_report", - { enabled: false }, - ); - }); - it("ensures the issues table syncs with full_refresh for github before enabling", async () => { const client = fakeClient(); const service = new SignalSourceService(); diff --git a/packages/core/src/inbox/signalSourceService.ts b/packages/core/src/inbox/signalSourceService.ts index 34a57bb782..7a740170ec 100644 --- a/packages/core/src/inbox/signalSourceService.ts +++ b/packages/core/src/inbox/signalSourceService.ts @@ -37,9 +37,7 @@ const SOURCE_TYPE_MAP: Partial> = { conversations: "ticket", health_checks: "health_issue", session_replay: "session_analysis_cluster", - // `llm_analytics` also emits per-evaluation `evaluation` signals configured on the - // evaluations page; this toggle only governs the project-level `evaluation_report` source, - // matching the web app's AI observability source card. + // AI observability signals are always emitted per evaluation report. llm_analytics: "evaluation_report", ...Object.fromEntries( EXTERNAL_INBOX_SOURCES.map((s) => [s.product, s.recordKind]), @@ -75,13 +73,6 @@ const ALL_SOURCE_PRODUCTS: SignalSourceProduct[] = [ ...EXTERNAL_INBOX_SOURCES.map((s) => s.product), ]; -// Products whose enabled state must match a single, specific `source_type` rather than any -// config under the product — `llm_analytics` also carries per-evaluation `evaluation` configs -// that this toggle must ignore. -const TYPE_SCOPED_SOURCES: Partial> = { - llm_analytics: "evaluation_report", -}; - function isWarehouseSource(product: SignalSourceProduct): boolean { return product in DATA_WAREHOUSE_SOURCES; } @@ -122,12 +113,8 @@ export function computeSourceValues( ), ); } else { - const scopedType = TYPE_SCOPED_SOURCES[product]; result[product] = configs.some( - (c) => - c.source_product === product && - (!scopedType || c.source_type === scopedType) && - c.enabled, + (c) => c.source_product === product && c.enabled, ); } } @@ -312,13 +299,8 @@ export class SignalSourceService { enabled: boolean, configs: SignalSourceConfig[] | undefined, ): Promise { + const existing = configs?.find((c) => c.source_product === product); const sourceType = SOURCE_TYPE_MAP[product]; - const scopedType = TYPE_SCOPED_SOURCES[product]; - const existing = configs?.find( - (c) => - c.source_product === product && - (!scopedType || c.source_type === scopedType), - ); if (existing) { await client.updateSignalSourceConfig(projectId, existing.id, { enabled, diff --git a/packages/ui/src/features/inbox/hooks/useSignalSourceToggles.ts b/packages/ui/src/features/inbox/hooks/useSignalSourceToggles.ts index d35640d451..69ae012443 100644 --- a/packages/ui/src/features/inbox/hooks/useSignalSourceToggles.ts +++ b/packages/ui/src/features/inbox/hooks/useSignalSourceToggles.ts @@ -24,9 +24,7 @@ const SOURCE_TYPE_MAP: Partial> = { conversations: "ticket", health_checks: "health_issue", session_replay: "session_analysis_cluster", - // `llm_analytics` also emits per-evaluation `evaluation` signals configured on the - // evaluations page; this toggle only governs the project-level `evaluation_report` source, - // matching the web app's AI observability source card. + // AI observability signals are always emitted per evaluation report. llm_analytics: "evaluation_report", ...Object.fromEntries( EXTERNAL_INBOX_SOURCES.map((s) => [s.product, s.recordKind]), @@ -73,13 +71,6 @@ const ALL_SOURCE_PRODUCTS: SourceKey[] = [ ...EXTERNAL_INBOX_SOURCES.map((s) => s.product), ]; -// Products whose enabled state must match a single, specific `source_type` rather than any -// config under the product — `llm_analytics` also carries per-evaluation `evaluation` configs -// that this toggle must ignore. -const TYPE_SCOPED_SOURCES: Partial> = { - llm_analytics: "evaluation_report", -}; - function isSetupSourceProduct(product: SourceKey): boolean { return product in DATA_WAREHOUSE_SOURCES; } @@ -102,12 +93,8 @@ function computeValues( ), ); } else { - const scopedType = TYPE_SCOPED_SOURCES[product]; result[product] = configs.some( - (c) => - c.source_product === product && - (!scopedType || c.source_type === scopedType) && - c.enabled, + (c) => c.source_product === product && c.enabled, ); } } @@ -311,13 +298,8 @@ export function useSignalSourceToggles() { } } } else { + const existing = configs?.find((c) => c.source_product === product); const sourceType = SOURCE_TYPE_MAP[product]; - const scopedType = TYPE_SCOPED_SOURCES[product]; - const existing = configs?.find( - (c) => - c.source_product === product && - (!scopedType || c.source_type === scopedType), - ); if (existing) { await client.updateSignalSourceConfig(projectId, existing.id, { enabled, From 19b7b9293f0c3bb5865cca905ab18d62344b244a Mon Sep 17 00:00:00 2001 From: Radu Raicea Date: Thu, 30 Jul 2026 10:04:36 -0400 Subject: [PATCH 4/4] feat(inbox): point AI observability source to evaluations setup Signals only emit once evaluations exist, so the AI observability source card now says to set up evaluations and links to the ai-evals docs. Generated-By: PostHog Code Task-Id: 9725e5be-8b91-44f8-8c12-3419a5bf080c --- .../src/features/inbox/components/SignalSourceToggles.tsx | 6 +++--- .../ui/src/features/inbox/components/responderAgentMeta.ts | 7 ++++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/packages/ui/src/features/inbox/components/SignalSourceToggles.tsx b/packages/ui/src/features/inbox/components/SignalSourceToggles.tsx index 924163c6f6..bb0d7f3a3b 100644 --- a/packages/ui/src/features/inbox/components/SignalSourceToggles.tsx +++ b/packages/ui/src/features/inbox/components/SignalSourceToggles.tsx @@ -326,12 +326,12 @@ export function SignalSourceToggles({ } label="AI observability" - description="Quality problems in your AI features" + description="Quality problems in your AI features. Set up evaluations to start getting signals." checked={value.llm_analytics} onCheckedChange={toggleLlmAnalytics} disabled={disabled} - docsUrl="https://posthog.com/docs/ai-observability" - docsLabel="AI observability" + docsUrl="https://posthog.com/docs/ai-evals" + docsLabel="evaluations" /> diff --git a/packages/ui/src/features/inbox/components/responderAgentMeta.ts b/packages/ui/src/features/inbox/components/responderAgentMeta.ts index 51a6af9a3f..b47d1faa59 100644 --- a/packages/ui/src/features/inbox/components/responderAgentMeta.ts +++ b/packages/ui/src/features/inbox/components/responderAgentMeta.ts @@ -60,9 +60,10 @@ export const RESPONDER_AGENT_GROUPS: ResponderAgentGroup[] = [ source: "llm_analytics", sourceProduct: "llm_analytics", label: "AI observability", - description: "Quality problems in your AI features.", - docsUrl: "https://posthog.com/docs/ai-observability", - docsLabel: "AI observability", + description: + "Quality problems in your AI features. Set up evaluations to start getting signals.", + docsUrl: "https://posthog.com/docs/ai-evals", + docsLabel: "evaluations", }, ], },