PostHog · trunk-io · Jul 30, 2026 · Jul 30, 2026 · Jul 30, 2026 · Jul 30, 2026
diff --git a/packages/core/src/inbox/signalSourceService.test.ts b/packages/core/src/inbox/signalSourceService.test.ts
@@ -124,6 +124,17 @@ describe("SignalSourceService.toggleSource", () => {
     });
   });
 
+  it("creates an llm_analytics config with the evaluation_report source type", async () => {
+    const client = fakeClient();
+    const service = new SignalSourceService();
+    await service.toggleSource(client, 1, "llm_analytics", true, [], []);
+    expect(client.createSignalSourceConfig).toHaveBeenCalledWith(1, {
+      source_product: "llm_analytics",
+      source_type: "evaluation_report",
+      enabled: true,
+    });
+  });
+
   it("ensures the issues table syncs with full_refresh for github before enabling", async () => {
     const client = fakeClient();
     const service = new SignalSourceService();

diff --git a/packages/core/src/inbox/signalSourceService.ts b/packages/core/src/inbox/signalSourceService.ts
@@ -37,6 +37,8 @@ const SOURCE_TYPE_MAP: Partial<Record<SignalSourceProduct, SourceType>> = {
   conversations: "ticket",
   health_checks: "health_issue",
   session_replay: "session_analysis_cluster",
+  // AI observability signals are always emitted per evaluation report.
+  llm_analytics: "evaluation_report",
   ...Object.fromEntries(
     EXTERNAL_INBOX_SOURCES.map((s) => [s.product, s.recordKind]),
   ),
@@ -67,6 +69,7 @@ const ALL_SOURCE_PRODUCTS: SignalSourceProduct[] = [
   "error_tracking",
   "health_checks",
   "session_replay",
+  "llm_analytics",
   ...EXTERNAL_INBOX_SOURCES.map((s) => s.product),
 ];
 

diff --git a/packages/shared/src/inbox-types.ts b/packages/shared/src/inbox-types.ts
@@ -424,13 +424,10 @@ export type SourceProduct =
   | ExternalInboxSourceProduct;
 
 /**
- * Products that render as a toggle in the Self-driving sources modal. Excludes non-toggle
- * products (`llm_analytics`, `signals_scout`) that appear only as signal origins.
+ * Products that render as a toggle in the Self-driving sources modal. Excludes `signals_scout`,
+ * which appears only as a signal origin and is always on rather than user-toggled.
  */
-export type ToggleableSourceProduct = Exclude<
-  SourceProduct,
-  "llm_analytics" | "signals_scout"
->;
+export type ToggleableSourceProduct = Exclude<SourceProduct, "signals_scout">;
 
 /**
  * Every backend signal `source_type`: the PostHog-native types (alphabetical) plus the
@@ -439,6 +436,7 @@ export type ToggleableSourceProduct = Exclude<
 export type SourceType =
   | "cross_source_issue"
   | "evaluation"
+  | "evaluation_report"
   | "health_issue"
   | "issue_created"
   | "issue_reopened"

diff --git a/packages/ui/src/features/inbox/components/ConfigureAgentsSection.tsx b/packages/ui/src/features/inbox/components/ConfigureAgentsSection.tsx
@@ -84,7 +84,6 @@ export function ConfigureAgentsSection() {
     handleSetupCancel,
     userAutonomyConfig,
     userAutonomyConfigLoading,
-    evaluationsUrl,
   } = useSignalSourceManager();
   const { hasGithubIntegration, isLoadingIntegrations } =
     useRepositoryIntegration();
@@ -188,7 +187,6 @@ export function ConfigureAgentsSection() {
                   disabled={!hasGithubIntegration}
                   sourceStates={sourceStates}
                   onSetup={handleSetup}
-                  evaluationsUrl={evaluationsUrl}
                 />
               )}
             </Box>

diff --git a/packages/ui/src/features/inbox/components/ResponderAgentRoster.tsx b/packages/ui/src/features/inbox/components/ResponderAgentRoster.tsx
@@ -14,7 +14,7 @@ import type { SignalSourceValues } from "@posthog/ui/features/inbox/components/S
 import { InboxBadge } from "@posthog/ui/features/inbox/components/utils/InboxBadge";
 import { getSourceProductMeta } from "@posthog/ui/features/inbox/components/utils/source-product-icons";
 import { Badge } from "@posthog/ui/primitives/Badge";
-import { Box, Flex, Spinner, Switch, Text, Tooltip } from "@radix-ui/themes";
+import { Box, Flex, Spinner, Switch, Text } from "@radix-ui/themes";
 import { type ComponentType, memo, useCallback } from "react";
 
 type AgentRosterStatus = "standby" | "watching" | "syncing" | "sync_failed";
@@ -61,7 +61,6 @@ interface ResponderAgentRosterProps {
     >
   >;
   onSetup?: (source: ResponderAgentSource) => void;
-  evaluationsUrl?: string;
 }
 
 export function ResponderAgentRoster({
@@ -70,7 +69,6 @@ export function ResponderAgentRoster({
   disabled,
   sourceStates,
   onSetup,
-  evaluationsUrl,
 }: ResponderAgentRosterProps) {
   return (
     <Flex direction="column" gap="5">
@@ -91,9 +89,6 @@ export function ResponderAgentRoster({
                 onSetup={onSetup}
               />
             ))}
-            {group.label === "PostHog data" && evaluationsUrl ? (
-              <EvaluationsAgentCard evaluationsUrl={evaluationsUrl} />
-            ) : null}
           </Box>
         </Flex>
       ))}
@@ -264,80 +259,6 @@ function AgentIcon({
   );
 }
 
-const EvaluationsAgentCard = memo(function EvaluationsAgentCard({
-  evaluationsUrl,
-}: {
-  evaluationsUrl: string;
-}) {
-  return (
-    <Box
-      onClick={() => window.open(evaluationsUrl, "_blank", "noopener")}
-      className={[
-        AGENT_CARD_BASE_CLASS,
-        "border-border",
-        AGENT_CARD_INTERACTIVE_IDLE_CLASS,
-      ].join(" ")}
-    >
-      <Flex align="start" justify="between" gap="3">
-        <Flex align="start" gap="3" className="min-w-0 flex-1">
-          <AgentIcon
-            accentColor="var(--purple-9)"
-            Icon={getSourceProductMeta("llm_analytics")?.Icon}
-          />
-          <Flex direction="column" gap="1" className="min-w-0">
-            <Flex align="center" gap="2" wrap="wrap">
-              <Text className="font-medium text-[13px] text-gray-12">
-                AI Observability
-              </Text>
-              <Tooltip content="This is only visible to staff users of PostHog">
-                <Badge color="blue" className="text-[11px]">
-                  Internal
-                </Badge>
-              </Tooltip>
-            </Flex>
-            <Text className="text-[13px] text-gray-11 leading-snug">
-              Quality problems in your AI features.
-            </Text>
-            <Text className="text-[13px] text-gray-11">
-              <a
-                href="https://posthog.com/docs/ai-observability"
-                target="_blank"
-                rel="noopener noreferrer"
-                onClick={(e) => {
-                  e.stopPropagation();
-                  e.preventDefault();
-                  window.open(
-                    "https://posthog.com/docs/ai-observability",
-                    "_blank",
-                    "noopener",
-                  );
-                }}
-                className="inline-flex items-center gap-1 text-(--accent-11) no-underline"
-              >
-                Learn about AI observability
-                <ArrowSquareOutIcon size={11} />
-              </a>
-            </Text>
-          </Flex>
-        </Flex>
-        <Button
-          type="button"
-          variant="primary"
-          size="sm"
-          className="shrink-0"
-          onClick={(e) => {
-            e.stopPropagation();
-            window.open(evaluationsUrl, "_blank", "noopener");
-          }}
-        >
-          Open
-          <ArrowSquareOutIcon size={12} />
-        </Button>
-      </Flex>
-    </Box>
-  );
-});
-
 function ResponderAgentCardSkeleton() {
   return (
     <Box className={`${AGENT_CARD_BASE_CLASS} border-border`}>

diff --git a/packages/ui/src/features/inbox/components/SignalSourceToggles.tsx b/packages/ui/src/features/inbox/components/SignalSourceToggles.tsx
@@ -16,7 +16,7 @@ import {
 } from "@posthog/shared";
 import { getSourceProductMeta } from "@posthog/ui/features/inbox/components/utils/source-product-icons";
 import { Badge } from "@posthog/ui/primitives/Badge";
-import { Box, Flex, Spinner, Switch, Text, Tooltip } from "@radix-ui/themes";
+import { Box, Flex, Spinner, Switch, Text } from "@radix-ui/themes";
 import { memo, useCallback } from "react";
 
 export type SignalSourceValues = Record<ToggleableSourceProduct, boolean>;
@@ -210,75 +210,6 @@ const ExternalSourceCard = memo(function ExternalSourceCard({
   );
 });
 
-interface EvaluationsSectionProps {
-  evaluationsUrl: string;
-}
-
-export const EvaluationsSection = memo(function EvaluationsSection({
-  evaluationsUrl,
-}: EvaluationsSectionProps) {
-  return (
-    <Box
-      p="3"
-      onClick={() => window.open(evaluationsUrl, "_blank", "noopener")}
-      className="cursor-pointer rounded-(--radius-3) border border-border bg-(--color-panel-solid) transition duration-150 hover:border-(--gray-6) hover:bg-(--gray-2) hover:shadow-sm"
-    >
-      <Flex align="center" justify="between" gap="4">
-        <Flex align="center" gap="3">
-          <Box className="shrink-0 text-gray-11">
-            <BrainIcon size={20} />
-          </Box>
-          <Flex direction="column" gap="1">
-            <Flex align="center" gap="2">
-              <Text className="font-medium text-gray-12 text-sm">
-                AI observability
-              </Text>
-              <Tooltip content="This is only visible to staff users of PostHog">
-                <Badge color="blue">Internal</Badge>
-              </Tooltip>
-            </Flex>
-            <Text className="text-[13px] text-gray-11">
-              Monitor how your AI features are performing
-            </Text>
-            <Text className="text-[13px] text-gray-11">
-              <a
-                href="https://posthog.com/docs/ai-observability"
-                target="_blank"
-                rel="noopener noreferrer"
-                onClick={(e) => {
-                  e.stopPropagation();
-                  e.preventDefault();
-                  window.open(
-                    "https://posthog.com/docs/ai-observability",
-                    "_blank",
-                    "noopener",
-                  );
-                }}
-                className="inline-flex items-center gap-[4px] text-(--accent-11) no-underline"
-              >
-                Learn about AI observability
-                <ArrowSquareOutIcon size={11} />
-              </a>
-            </Text>
-          </Flex>
-        </Flex>
-        <Button
-          type="button"
-          variant="primary"
-          size="sm"
-          onClick={(e) => {
-            e.stopPropagation();
-            window.open(evaluationsUrl, "_blank", "noopener");
-          }}
-        >
-          Open
-          <ArrowSquareOutIcon size={12} />
-        </Button>
-      </Flex>
-    </Box>
-  );
-});
-
 function SourceRunningIndicator({
   status,
   message,
@@ -303,7 +234,6 @@ interface SignalSourceTogglesProps {
   disabled?: boolean;
   sourceStates?: Partial<Record<ToggleableSourceProduct, SourceState>>;
   onSetup?: (source: ToggleableSourceProduct) => void;
-  evaluationsUrl?: string;
 }
 
 export function SignalSourceToggles({
@@ -312,7 +242,6 @@ export function SignalSourceToggles({
   disabled,
   sourceStates,
   onSetup,
-  evaluationsUrl,
 }: SignalSourceTogglesProps) {
   const toggleSessionReplay = useCallback(
     (checked: boolean) => onToggle("session_replay", checked),
@@ -330,6 +259,10 @@ export function SignalSourceToggles({
     (checked: boolean) => onToggle("health_checks", checked),
     [onToggle],
   );
+  const toggleLlmAnalytics = useCallback(
+    (checked: boolean) => onToggle("llm_analytics", checked),
+    [onToggle],
+  );
 
   return (
     <Flex gap="4">
@@ -390,9 +323,16 @@ export function SignalSourceToggles({
               ) : undefined
             }
           />
-          {evaluationsUrl && (
-            <EvaluationsSection evaluationsUrl={evaluationsUrl} />
-          )}
+          <SignalSourceToggleCard
+            icon={<BrainIcon size={20} />}
+            label="AI observability"
+            description="Quality problems in your AI features. Set up evaluations to start getting signals."
+            checked={value.llm_analytics}
+            onCheckedChange={toggleLlmAnalytics}
+            disabled={disabled}
+            docsUrl="https://posthog.com/docs/ai-evals"
+            docsLabel="evaluations"
+          />
         </Flex>
       </Flex>
 

diff --git a/packages/ui/src/features/inbox/components/responderAgentMeta.ts b/packages/ui/src/features/inbox/components/responderAgentMeta.ts
@@ -56,6 +56,15 @@ export const RESPONDER_AGENT_GROUPS: ResponderAgentGroup[] = [
         docsLabel: "Session Replay",
         alpha: true,
       },
+      {
+        source: "llm_analytics",
+        sourceProduct: "llm_analytics",
+        label: "AI observability",
+        description:
+          "Quality problems in your AI features. Set up evaluations to start getting signals.",
+        docsUrl: "https://posthog.com/docs/ai-evals",
+        docsLabel: "evaluations",
+      },
     ],
   },
   {

diff --git a/packages/ui/src/features/inbox/hooks/useSignalSourceToggles.ts b/packages/ui/src/features/inbox/hooks/useSignalSourceToggles.ts
@@ -24,6 +24,8 @@ const SOURCE_TYPE_MAP: Partial<Record<SourceKey, SourceType>> = {
   conversations: "ticket",
   health_checks: "health_issue",
   session_replay: "session_analysis_cluster",
+  // AI observability signals are always emitted per evaluation report.
+  llm_analytics: "evaluation_report",
   ...Object.fromEntries(
     EXTERNAL_INBOX_SOURCES.map((s) => [s.product, s.recordKind]),
   ),
@@ -40,6 +42,7 @@ const SOURCE_LABELS: Partial<Record<SourceKey, string>> = {
   error_tracking: "Error tracking",
   health_checks: "Health checks",
   session_replay: "Session replay",
+  llm_analytics: "AI observability",
   ...Object.fromEntries(
     EXTERNAL_INBOX_SOURCES.map((s) => [s.product, s.label]),
   ),
@@ -64,6 +67,7 @@ const ALL_SOURCE_PRODUCTS: SourceKey[] = [
   "error_tracking",
   "health_checks",
   "session_replay",
+  "llm_analytics",
   ...EXTERNAL_INBOX_SOURCES.map((s) => s.product),
 ];