Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 26 additions & 17 deletions packages/ai/src/route/client.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import { Endpoint, type EndpointPatch } from "./endpoint"
import { RequestExecutor } from "./executor"
import { Framing } from "./framing"
import { HttpTransport } from "./transport"
import type { Transport, TransportRuntime } from "./transport"
import type { HttpRequestTransform, Transport, TransportRuntime } from "./transport"
import { WebSocketExecutor } from "./transport"
import type { Protocol } from "./protocol"
import { applyCachePolicy } from "../cache-policy"
Expand Down Expand Up @@ -46,7 +46,11 @@ export interface Route<Body, Prepared = unknown> {
readonly body: RouteBody<Body>
readonly with: (patch: RoutePatch<Body, Prepared>) => Route<Body, Prepared>
readonly model: <Options extends ProviderOptions = ProviderOptions>(input: RouteMappedModelInput) => Model<Options>
readonly prepareTransport: (body: Body, request: LLMRequest) => Effect.Effect<Prepared, LLMError>
readonly prepareTransport: (
body: Body,
request: LLMRequest,
options?: StreamOptions,
) => Effect.Effect<Prepared, LLMError>
readonly streamPrepared: (
prepared: Prepared,
request: LLMRequest,
Expand Down Expand Up @@ -145,12 +149,16 @@ export interface Interface {
readonly generate: GenerateMethod
}

export interface StreamOptions {
readonly transform?: HttpRequestTransform
}

export interface StreamMethod {
(request: LLMRequest): Stream.Stream<LLMEvent, LLMError>
(request: LLMRequest, options?: StreamOptions): Stream.Stream<LLMEvent, LLMError>
}

export interface GenerateMethod {
(request: LLMRequest): Effect.Effect<LLMResponse, LLMError>
(request: LLMRequest, options?: StreamOptions): Effect.Effect<LLMResponse, LLMError>
}

export class Service extends Context.Service<Service, Interface>()("@opencode/LLMClient") {}
Expand Down Expand Up @@ -286,14 +294,15 @@ function makeFromTransport<Body, Prepared, Frame, Event, State>(
},
model: <Options extends ProviderOptions = ProviderOptions>(input: RouteMappedModelInput) =>
makeRouteModel<Options>(route, input),
prepareTransport: (body, request) =>
prepareTransport: (body, request, options) =>
routeInput.transport.prepare({
body,
request,
endpoint: routeInput.endpoint,
auth: routeInput.auth ?? Auth.none,
encodeBody,
headers: routeInput.headers,
transform: options?.transform,
}),
streamPrepared: (prepared: Prepared, request: LLMRequest, runtime: TransportRuntime) => {
const route = `${request.model.provider}/${request.model.route.id}`
Expand Down Expand Up @@ -359,14 +368,14 @@ export function make<Body, Prepared, Frame, Event, State>(
})
}

const compile = Effect.fn("LLM.compile")(function* (request: LLMRequest) {
const compile = Effect.fn("LLM.compile")(function* (request: LLMRequest, options?: StreamOptions) {
const resolved = applyCachePolicy(resolveRequestOptions(request))
const route = resolved.model.route

const body = yield* route.body
.from(resolved)
.pipe(Effect.flatMap(ProviderShared.validateWith(Schema.decodeUnknownEffect(route.body.schema))))
const prepared = yield* route.prepareTransport(body, resolved)
const prepared = yield* route.prepareTransport(body, resolved, options)

return {
request: resolved,
Expand All @@ -389,17 +398,17 @@ export const compileRequest = Effect.fn("LLM.compileRequest")(function* (request
}
})

const streamRequestWith = (runtime: TransportRuntime) => (request: LLMRequest) =>
const streamRequestWith = (runtime: TransportRuntime) => (request: LLMRequest, options?: StreamOptions) =>
Stream.unwrap(
Effect.gen(function* () {
const compiled = yield* compile(request)
const compiled = yield* compile(request, options)
return compiled.route.streamPrepared(compiled.prepared, compiled.request, runtime)
}),
)

const generateWith = (stream: Interface["stream"]) =>
Effect.fn("LLM.generate")(function* (request: LLMRequest) {
const state = yield* stream(request).pipe(Stream.runFold(LLMResponse.empty, LLMResponse.reduce))
Effect.fn("LLM.generate")(function* (request: LLMRequest, options?: StreamOptions) {
const state = yield* stream(request, options).pipe(Stream.runFold(LLMResponse.empty, LLMResponse.reduce))
const response = LLMResponse.complete(state)
if (response) return response
return yield* ProviderShared.eventError(
Expand All @@ -408,24 +417,24 @@ const generateWith = (stream: Interface["stream"]) =>
)
})

export function stream(request: LLMRequest): Stream.Stream<LLMEvent, LLMError> {
export function stream(request: LLMRequest, options?: StreamOptions): Stream.Stream<LLMEvent, LLMError> {
return Stream.unwrap(
Effect.gen(function* () {
return (yield* Service).stream(request)
return (yield* Service).stream(request, options)
}),
) as Stream.Stream<LLMEvent, LLMError>
}

export function generate(request: LLMRequest): Effect.Effect<LLMResponse, LLMError> {
export function generate(request: LLMRequest, options?: StreamOptions): Effect.Effect<LLMResponse, LLMError> {
return Effect.gen(function* () {
return yield* (yield* Service).generate(request)
return yield* (yield* Service).generate(request, options)
}) as Effect.Effect<LLMResponse, LLMError>
}

export const streamRequest = (request: LLMRequest) =>
export const streamRequest = (request: LLMRequest, options?: StreamOptions) =>
Stream.unwrap(
Effect.gen(function* () {
return (yield* Service).stream(request)
return (yield* Service).stream(request, options)
}),
)

Expand Down
3 changes: 2 additions & 1 deletion packages/ai/src/route/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ export type {
AnyRoute,
Interface as LLMClientShape,
Service as LLMClientService,
StreamOptions,
} from "./client"
export * from "./executor"
export { Auth } from "./auth"
Expand All @@ -22,4 +23,4 @@ export type { ApiKeyMode, AuthOverride, ProviderAuthOption } from "./auth-option
export type { Definition as EndpointFn, EndpointInput } from "./endpoint"
export type { Definition as FramingDef } from "./framing"
export type { Protocol as ProtocolDef } from "./protocol"
export type { Transport as TransportDef, TransportRuntime } from "./transport"
export type { HttpRequest, HttpRequestTransform, Transport as TransportDef, TransportRuntime } from "./transport"
19 changes: 12 additions & 7 deletions packages/ai/src/route/transport/http.ts
Original file line number Diff line number Diff line change
Expand Up @@ -120,14 +120,19 @@ export const httpJson = <Body, Frame>(input: HttpJsonInput<Body, Frame>): HttpJs
id: "http-json",
with: (patch) => httpJson({ ...input, ...patch }),
prepare: (prepareInput) =>
jsonRequestParts({
...prepareInput,
}).pipe(
Effect.map((parts) => ({
request: ProviderShared.jsonPost({ url: parts.url, body: parts.bodyText, headers: parts.headers }),
Effect.gen(function* () {
const parts = yield* jsonRequestParts({ ...prepareInput })
const request = { url: parts.url, method: "POST", headers: { ...parts.headers }, body: parts.bodyText }
yield* (prepareInput.transform?.(request) ?? Effect.void)
return {
request: ProviderShared.jsonPost({
url: request.url,
body: request.body ?? "",
headers: Headers.fromInput(request.headers),
}),
framing: input.framing,
})),
),
}
}),
frames: (prepared, request, runtime) =>
Stream.unwrap(
runtime.http
Expand Down
10 changes: 10 additions & 0 deletions packages/ai/src/route/transport/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,15 @@ export interface TransportRuntime {
readonly webSocket?: WebSocketExecutorInterface
}

export interface HttpRequest {
url: string
readonly method: string
headers: Record<string, string>
body: string | undefined
}

export type HttpRequestTransform = (request: HttpRequest) => Effect.Effect<void>

export interface Transport<Body, Prepared, Frame> {
readonly id: string
readonly prepare: (input: TransportPrepareInput<Body>) => Effect.Effect<Prepared, LLMError>
Expand All @@ -27,6 +36,7 @@ export interface TransportPrepareInput<Body> {
readonly auth: Auth.Definition
readonly encodeBody: (body: Body) => string
readonly headers?: (input: { readonly request: LLMRequest }) => Record<string, string>
readonly transform?: HttpRequestTransform
}

export * as HttpTransport from "./http"
Expand Down
34 changes: 34 additions & 0 deletions packages/ai/test/compile.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,40 @@ describe("request option precedence", () => {
),
)

it.effect("transforms the final HTTP request after serialization and authentication", () =>
LLMClient.generate(
LLM.request({
model: OpenAIChat.route
.with({ endpoint: { baseURL: "https://api.openai.test/v1/" }, auth: Auth.bearer("fresh-key") })
.model({ id: "gpt-4o-mini" }),
prompt: "Say hello.",
}),
{
transform: (request) =>
Effect.sync(() => {
expect(request.headers.authorization).toBe("Bearer fresh-key")
request.url = "https://proxy.test/v1/chat/completions"
request.headers["x-plugin"] = "transformed"
request.body = JSON.stringify({ transformed: true })
}),
},
).pipe(
Effect.provide(
dynamicResponse((input) =>
Effect.gen(function* () {
const web = yield* HttpClientRequest.toWeb(input.request).pipe(Effect.orDie)
expect(web.url).toBe("https://proxy.test/v1/chat/completions")
expect(web.headers.get("x-plugin")).toBe("transformed")
expect(decodeJson(input.text)).toEqual({ transformed: true })
return input.respond(sseEvents(deltaChunk({}, "stop")), {
headers: { "content-type": "text/event-stream" },
})
}),
),
),
),
)

it.effect("rejects raw body overlays for protocol-owned roots", () =>
Effect.gen(function* () {
const model = OpenAIChat.route
Expand Down
23 changes: 23 additions & 0 deletions packages/core/src/session/model-request.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
export * as SessionModelRequest from "./model-request"

import { LLM, Message, SystemPart, type LLMRequest } from "@opencode-ai/ai"
import type { StreamOptions } from "@opencode-ai/ai/route"
import type { Content } from "@opencode-ai/schema/tool"
import { SessionError } from "@opencode-ai/schema/session-error"
import { Cause, Config, Context, Effect, Layer, Result } from "effect"
Expand Down Expand Up @@ -37,6 +38,7 @@ const declineDefect = (cause: Cause.Cause<Tool.Error>) => {

interface Prepared {
readonly request: LLMRequest
readonly options: StreamOptions
/**
* One request-scoped execution operation. Unknown, hook-removed, and
* step-limit-violating calls fail individually through the same seam.
Expand Down Expand Up @@ -162,6 +164,26 @@ export const layer = Layer.effect(
tools: hookedTools,
toolChoice: stepLimitReached ? "none" : undefined,
})
const options: StreamOptions = {
transform: (request) =>
hooks
.trigger("session", "request", {
sessionID: session.id,
agent: agent.id,
model: resolved.ref,
...request,
})
.pipe(
Effect.tap((event) =>
Effect.sync(() => {
request.url = event.url
request.headers = event.headers
request.body = event.body
}),
),
Effect.asVoid,
),
}
if (promptCacheSnapshots) {
const current = PromptCacheDiagnostics.snapshot(request)
const comparison = PromptCacheDiagnostics.compare(promptCacheSnapshots.get(session.id), current)
Expand Down Expand Up @@ -190,6 +212,7 @@ export const layer = Layer.effect(
}
return {
request,
options,
executeTool,
stepLimitReached,
}
Expand Down
2 changes: 1 addition & 1 deletion packages/core/src/session/runner/llm.ts
Original file line number Diff line number Diff line change
Expand Up @@ -279,7 +279,7 @@ const layer = Layer.effect(
// event durably, fork one fiber per local tool call, and hold back a virgin
// context-overflow provider error so settlement may recover it via compaction.
let overflowFailure: ProviderErrorEvent | undefined
const providerStream = llm.stream(prepared.request).pipe(
const providerStream = llm.stream(prepared.request, prepared.options).pipe(
Stream.runForEach((event) =>
Effect.gen(function* () {
if (overflowFailure || publisher.hasProviderError()) return
Expand Down
8 changes: 8 additions & 0 deletions packages/plugin/src/effect/session.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import type { SessionApi } from "@opencode-ai/client/effect/api"
import type { Message, SystemPart } from "@opencode-ai/ai"
import type { HttpRequest } from "@opencode-ai/ai/route"
import type { Agent } from "@opencode-ai/schema/agent"
import type { Model } from "@opencode-ai/schema/model"
import type { Session } from "@opencode-ai/schema/session"
Expand All @@ -15,8 +16,15 @@ export interface SessionContext {
tools: Record<string, { description: string; input: JsonSchema.JsonSchema }>
}

export interface SessionRequest extends HttpRequest {
readonly sessionID: Session.ID
readonly agent: Agent.ID
readonly model: Model.Ref
}

export interface SessionHooks {
readonly context: SessionContext
readonly request: SessionRequest
}

export type SessionDomain = Pick<
Expand Down
8 changes: 8 additions & 0 deletions packages/plugin/src/promise/session.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import type { SessionApi } from "@opencode-ai/client/promise/api"
import type { Message, SystemPart } from "@opencode-ai/ai"
import type { HttpRequest } from "@opencode-ai/ai/route"
import type { Agent } from "@opencode-ai/schema/agent"
import type { Model } from "@opencode-ai/schema/model"
import type { Session } from "@opencode-ai/schema/session"
Expand All @@ -15,8 +16,15 @@ export interface SessionContext {
tools: Record<string, { description: string; input: JsonSchema.JsonSchema }>
}

export interface SessionRequest extends HttpRequest {
readonly sessionID: Session.ID
readonly agent: Agent.ID
readonly model: Model.Ref
}

export interface SessionHooks {
readonly context: SessionContext
readonly request: SessionRequest
}

export type SessionDomain = Pick<
Expand Down
Loading