[pull] main from danny-avila:main

AGENTS.md

@@ -4,17 +4,18 @@
 
 `@librechat/agents` is a TypeScript library for LLM agent orchestration — tool calling, multi-agent graphs, message formatting, streaming, and provider abstraction (Anthropic, Bedrock, VertexAI, OpenAI, Google). Published as `@librechat/agents` on npm. This is a major backend dependency of [LibreChat](../LibreChat/CLAUDE.md) (same team).
 
-| Path            | Purpose                                             |
-| --------------- | --------------------------------------------------- |
-| `src/messages/` | Message formatting, caching, content processing     |
-| `src/graphs/`   | LangGraph-based agent graphs (single + multi-agent) |
-| `src/llm/`      | Provider-specific LLM wrappers and utilities        |
-| `src/tools/`    | Tool definitions and search                         |
-| `src/agents/`   | Agent definitions and handoff logic                 |
-| `src/types/`    | Shared TypeScript types                             |
-| `src/common/`   | Enums, constants                                    |
-| `src/run.ts`    | Main run orchestration                              |
-| `src/stream.ts` | Streaming logic                                     |
+| Path                                         | Purpose                                                                              |
+| -------------------------------------------- | ------------------------------------------------------------------------------------ |
+| `src/messages/`                              | Message formatting, caching, content processing                                      |
+| `src/graphs/`                                | LangGraph-based agent graphs (single + multi-agent)                                  |
+| `src/llm/`                                   | Provider-specific LLM wrappers and utilities                                         |
+| `src/tools/`                                 | Tool definitions and search                                                          |
+| `src/agents/`                                | Agent definitions and handoff logic                                                  |
+| `src/types/`                                 | Shared TypeScript types                                                              |
+| `src/common/`                                | Enums, constants                                                                     |
+| `src/run.ts`                                 | Main run orchestration                                                               |
+| `src/stream.ts`                              | Streaming logic                                                                      |
+| `src/langfuse*.ts`, `src/instrumentation.ts` | Langfuse tracing integration (see [Langfuse Trace Shaping](#langfuse-trace-shaping)) |
 
 ---
 
@@ -118,6 +119,44 @@ Multi-line imports count total character length across all lines. Consolidate va
 
 ---
 
+## Langfuse Trace Shaping
+
+This library is LibreChat's tracing surface: every agent run it orchestrates is exported to Langfuse, and trace quality is a product feature. Any change that touches graphs, node naming, callbacks, tool execution, message serialization, streaming, or providers must keep traces well-shaped — ask "how will this look in Langfuse?" as part of the change, not after.
+
+### Module Map
+
+| Module                                                         | Responsibility                                                               |
+| -------------------------------------------------------------- | ---------------------------------------------------------------------------- |
+| `src/langfuseTraceShaping.ts`                                  | Export-time span rename/retype/drop rules (the shape itself)                 |
+| `src/langfuseToolOutputTracing.ts`                             | Span processor applying shaping hooks + tool-output redaction                |
+| `src/langfuse.ts`                                              | Callback handler, identity/tags/metadata, control-flow + usage normalization |
+| `src/instrumentation.ts`                                       | Tracer provider bootstrap, per-tenant routing, deterministic trace ids       |
+| `src/langfuseConfig.ts`                                        | Config resolution/merging (env vs run vs agent level)                        |
+| `src/langfuseRuntimeContext.ts`, `src/langfuseRuntimeScope.ts` | Per-run scoping so concurrent runs/tenants never cross-contaminate           |
+
+### Invariants — what "well-shaped" means
+
+These originated from direct Langfuse-team feedback (PRs #288, #316) and must survive all future changes:
+
+- **Stable, operation-describing span names.** `agent`, `tool-dispatch`, `llm` — never leak ephemeral agent ids (`agent=<provider__model>`) or provider class names (`ChatOpenAI`) into span names. The model belongs on the generation's model attribute; name-based logic downstream must not break when models switch.
+- **Correct observation types.** Agent trace roots and agent nodes are `agent` observations; tool dispatch is a `chain`; individual tool calls are `tool`; LLM calls are `generation`; title trace roots are `chain`.
+- **No plumbing noise.** LangGraph internals (`__start__` channel seeds, anonymous `RunnableLambda` pass-throughs) are dropped at export. A trace tree should read like the run's story — if a new node adds noise, extend the shaping/drop rules rather than shipping it raw.
+- **Trace input/output are the conversation, not the state.** Root input reduces to the user's question, output to the assistant's answer — never full serialized graph state. Tool-dispatch input is scoped to the pending tool calls.
+- **Control flow is not an error.** `GraphInterrupt` and `ParentCommand` end their traces as successful with `controlFlow` outputs, not as error traces.
+- **Usage/cost is accurate per provider.** e.g. Bedrock cache read/write tokens are folded into input tokens so Langfuse cost math is right.
+- **Redaction is honored everywhere tool output can surface** — tool spans, and any generation input that embeds tool results (e.g. the activity-label prompt).
+- **Identity and metadata always propagate**: `userId`, `sessionId`, tags, environment, and trace metadata (`messageId`, `parentMessageId`, `agentId`, `agentName`) — including across LangChain callbacks that fire outside the caller's OTEL context.
+- **Trace identity is self-contained.** Root observations never inherit trace ids or parents from foreign ambient OTEL spans (e.g. a host's HTTP auto-instrumentation): the callback handler detaches them so roots stay true roots, deterministic ids apply, and concurrent runs inside one request context (an agent run plus a title run) cannot merge into one trace. Spans created through the Langfuse tracer provider are honored as parents, so hosts can still group runs under their own Langfuse observations deliberately.
+- **Deterministic trace ids** when a run opts in (`LangfuseConfig.deterministicTraceId`), so host apps can attach scores/feedback by regenerating the id from the run id.
+
+### Verifying
+
+- Run the tracing specs after any change in this area: `npx jest langfuse deterministic-trace-id` (specs live in `src/specs/langfuse-*.test.ts` and `src/tools/__tests__/ToolNode.langfuse.test.ts`). New shaping rules get spec coverage in `langfuse-trace-shaping.test.ts`.
+- For structural changes, verify against a real Langfuse project: set `LANGFUSE_PUBLIC_KEY`/`LANGFUSE_SECRET_KEY`/`LANGFUSE_BASE_URL`, run an agent, and inspect the observation tree in the UI or via `GET /api/public/traces` — confirm span names, observation types, and root input/output match the invariants above.
+- For nontrivial Langfuse SDK work, use Langfuse's own agent-facing resources instead of guessing SDK behavior: the [Langfuse skill](https://github.com/langfuse/skills/tree/main/skills/langfuse), the docs MCP server (`https://langfuse.com/api/mcp`), and [llms.txt](https://langfuse.com/llms.txt).
+
+---
+
 ## Formatting
 
 Fix all formatting lint errors (trailing spaces, tabs, newlines, indentation) using auto-fix when available. All TypeScript/ESLint warnings and errors **must** be resolved.

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@librechat/agents",
-  "version": "3.3.7",
+  "version": "3.3.8",
   "main": "./dist/cjs/main.cjs",
   "module": "./dist/esm/main.mjs",
   "types": "./dist/types/index.d.ts",
@@ -234,7 +234,6 @@
     "@langfuse/tracing": "^5.4.1",
     "@opentelemetry/context-async-hooks": "^2.9.0",
     "@opentelemetry/sdk-node": "^0.220.0",
-    "@scarf/scarf": "^1.4.0",
     "@types/diff": "^7.0.2",
     "ai-tokenizer": "^1.0.6",
     "axios": "^1.18.1",
@@ -246,6 +245,7 @@
     "nanoid": "^3.3.7",
     "okapibm25": "^1.4.1",
     "openai": "^6.46.0",
+    "reo-census": "^1.2.9",
     "uuid": "^11.1.1"
   },
   "peerDependencies": {

src/graphs/MultiAgentGraph.ts

@@ -27,6 +27,20 @@
 const HANDOFF_INSTRUCTIONS_PATTERN = /(?:Instructions?|Context):\s*(.+)/is;
 const HANDOFF_INSTRUCTIONS_KEY = 'handoff_instructions';
 
+/**
+ * Handoff and fan-in prompts that route work between agents. Built in-run and
+ * never persisted as standalone payload entries, so they are marked synthetic:
+ * `messagesStateReducer` would otherwise give them a plain UUID and downstream
+ * consumers — compaction coverage anchors — could not tell them apart from a
+ * message replayed out of the payload.
+ */
+function buildRoutingPrompt(content: string): HumanMessage {
+  return new HumanMessage({
+    content,
+    additional_kwargs: { role: 'user', isMeta: true, source: 'routing' },
+  });
+}
+
 function getHandoffInstructions(
   input: Record<string, unknown>,
   promptKey: string,
@@ -528,7 +542,7 @@
                * 3. Include all messages before the AIMessage plus the filtered pair
                */
               const messages = state.messages;
               let filteredMessages = messages;
               let aiMessageIndex = -1;
 
               /** Find the AIMessage containing this tool call */
@@ -986,12 +1000,12 @@
                 new AIMessage(
                   `[Processed tool result and transferring to ${agentId}]`
                 ),
-                new HumanMessage(instructions),
+                buildRoutingPrompt(instructions),
               ];
             } else {
               messagesForAgent = [
                 ...filteredMessages,
-                new HumanMessage(instructions),
+                buildRoutingPrompt(instructions),
               ];
             }
           }
@@ -1204,15 +1218,15 @@
               effectiveExcludeResults === false
             ) {
               return {
-                messages: [new HumanMessage(promptText)],
+                messages: [buildRoutingPrompt(promptText)],
               };
             }
 
             /** When `excludeResults` is true, use agentMessages channel
              * to pass filtered messages + prompt to the destination agent
              */
             const filteredMessages = state.messages.slice(0, this.startIndex);
-            const promptMessage = new HumanMessage(promptText);
+            const promptMessage = buildRoutingPrompt(promptText);
             return {
               messages: [promptMessage],
               agentMessages: messagesStateReducer(filteredMessages, [

src/instrumentation.ts

@@ -1,4 +1,4 @@
-import { createHash, randomBytes } from 'node:crypto';
+import { randomBytes } from 'node:crypto';
 import { setLangfuseTracerProvider } from '@langfuse/tracing';
 import { BasicTracerProvider } from '@opentelemetry/sdk-trace-base';
 import { context, ROOT_CONTEXT, createContextKey } from '@opentelemetry/api';
@@ -13,16 +13,16 @@ import type { LangfuseSpanProcessorParams } from '@langfuse/otel';
 import type { Context } from '@opentelemetry/api';
 import type * as t from '@/types';
 import {
-  createLibreChatTraceAttributes,
-  hasLangfuseConfigCredentials,
-  hasLangfuseEnvCredentials,
-  hasLangfuseEnvConfig,
-} from '@/langfuse';
+  getLangfuseDestinationKey,
+  getLangfuseSpanProcessorParams,
+  registerLangfuseManagedSpan,
+} from '@/langfuseSpanRegistry';
 import {
   resolveLangfuseConfigForSpan,
   resolveTraceIdSeedForSpan,
 } from '@/langfuseRuntimeScope';
 import { createLangfuseSpanProcessor } from '@/langfuseToolOutputTracing';
+import { createLibreChatTraceAttributes } from '@/langfuse';
 import { traceIdFromSeed } from '@/langfuseRuntimeContext';
 import { isPresent } from '@/utils/misc';
 
@@ -75,75 +75,15 @@ export function ensureOpenTelemetryContextManager(): void {
   }
 }
 
-function resolveLangfuseEnvironment(
-  langfuse?: t.LangfuseConfig
-): string | undefined {
-  const candidates = [
-    langfuse?.environment,
-    process.env.LANGFUSE_TRACING_ENVIRONMENT,
-    process.env.NODE_ENV,
-  ];
-  for (const candidate of candidates) {
-    if (candidate != null && candidate.trim() !== '') {
-      return candidate.trim();
-    }
-  }
-  return undefined;
-}
-
-function getLangfuseSpanProcessorParams(
-  langfuse?: t.LangfuseConfig
-): LangfuseSpanProcessorParams | undefined {
-  if (langfuse?.enabled === false) {
-    return undefined;
-  }
-  const environment = resolveLangfuseEnvironment(langfuse);
-  if (hasLangfuseConfigCredentials(langfuse)) {
-    return {
-      publicKey: langfuse.publicKey,
-      secretKey: langfuse.secretKey,
-      ...(isPresent(langfuse.baseUrl) ? { baseUrl: langfuse.baseUrl } : {}),
-      ...(isPresent(environment) ? { environment } : {}),
-    };
-  }
-  if (hasLangfuseEnvConfig()) {
-    const baseUrl =
-      langfuse?.baseUrl ??
-      process.env.LANGFUSE_BASE_URL ??
-      process.env.LANGFUSE_BASEURL;
-    return {
-      publicKey: process.env.LANGFUSE_PUBLIC_KEY as string,
-      secretKey: process.env.LANGFUSE_SECRET_KEY as string,
-      ...(isPresent(baseUrl) ? { baseUrl } : {}),
-      ...(isPresent(environment) ? { environment } : {}),
-    };
-  }
-  if (isPresent(langfuse?.baseUrl) && hasLangfuseEnvCredentials()) {
-    return {
-      publicKey: process.env.LANGFUSE_PUBLIC_KEY as string,
-      secretKey: process.env.LANGFUSE_SECRET_KEY as string,
-      baseUrl: langfuse.baseUrl,
-      ...(isPresent(environment) ? { environment } : {}),
-    };
-  }
-  return undefined;
-}
-
-function hashCacheKeyValue(value: string | undefined): string | undefined {
-  return isPresent(value)
-    ? createHash('sha256').update(value, 'utf8').digest('hex')
-    : undefined;
-}
-
-function getLangfuseTracerProviderKey(
-  params: LangfuseSpanProcessorParams,
+/** Cache key for processor instances: the destination plus processor-level
+ *  policy (`toolOutputTracing` is baked into each processor's redaction
+ *  behavior), unlike the pure destination identity used for span parenting. */
+function getLangfuseProcessorCacheKey(
+  destinationKey: string,
   langfuse?: t.LangfuseConfig
 ): string {
   return JSON.stringify({
-    publicKey: params.publicKey,
-    secretKeyHash: hashCacheKeyValue(params.secretKey),
-    baseUrl: params.baseUrl,
-    environment: params.environment,
+    destinationKey,
     toolOutputTracing: langfuse?.toolOutputTracing,
   });
 }
@@ -160,25 +100,43 @@ class RoutingLangfuseSpanProcessor implements SpanProcessor {
     if (params == null) {
       return undefined;
     }
+    return this.ensureProcessorForKey(
+      getLangfuseProcessorCacheKey(getLangfuseDestinationKey(params), langfuse),
+      params,
+      langfuse
+    );
+  }
 
-    const processorKey = getLangfuseTracerProviderKey(params, langfuse);
-    const existing = this.processors.get(processorKey);
+  private ensureProcessorForKey(
+    processorCacheKey: string,
+    params: LangfuseSpanProcessorParams,
+    langfuse?: t.LangfuseConfig
+  ): SpanProcessor {
+    const existing = this.processors.get(processorCacheKey);
     if (existing != null) {
       return existing;
     }
 
     const processor = createLangfuseSpanProcessor(params, langfuse);
-    this.processors.set(processorKey, processor);
+    this.processors.set(processorCacheKey, processor);
     return processor;
   }
 
   onStart(span: Span, parentContext: Context): void {
     const langfuse = resolveLangfuseConfigForSpan(parentContext);
-    const processor = this.ensureProcessor(langfuse);
-    if (processor == null) {
+    const params = getLangfuseSpanProcessorParams(langfuse);
+    if (params == null) {
       return;
     }
 
+    const destinationKey = getLangfuseDestinationKey(params);
+    const processor = this.ensureProcessorForKey(
+      getLangfuseProcessorCacheKey(destinationKey, langfuse),
+      params,
+      langfuse
+    );
+    registerLangfuseManagedSpan(span, destinationKey);
+
     const librechatTraceAttributes = createLibreChatTraceAttributes(
       langfuse?.librechatTraceAttributes ?? {}
     );