Skip to content

Commit c3b9011

Browse files
seawattsseawatts
authored
feat: add media (image/audio) support for React/Next.js (#1646)
Added new support for passing in images and audio natively to a baml react hook. See docs here: https://docs.boundaryml.com/ref/baml_client/media Fixes: #1540
1 parent 7d6b4cb commit c3b9011

File tree

104 files changed

+8047
-2581
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

104 files changed

+8047
-2581
lines changed

‎engine/language_client_codegen/src/typescript/templates/react/hooks.tsx.j2‎

Lines changed: 18 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,11 @@
11
'use client'
22

3-
import { BamlErrors, toBamlError } from '@boundaryml/baml/errors'
3+
import type { BamlErrors } from '@boundaryml/baml/errors'
4+
import { toBamlError } from '@boundaryml/baml/errors'
45
import { useCallback, useMemo, useReducer, useTransition } from 'react'
56
import * as Actions from './server'
67
import * as StreamingActions from './server_streaming'
7-
import { StreamingServerTypes } from './server_streaming_types'
8+
import type { StreamingServerTypes } from './server_streaming_types'
89

910
/**
1011
* Type representing a BAML stream response.
@@ -226,7 +227,18 @@ function useBamlAction<FunctionName extends FunctionNames>(
226227
try {
227228
let response: Awaited<ReturnType<ServerAction>>
228229
startTransition(async () => {
229-
response = await action(...input)
230+
// Transform any BamlImage or BamlAudio inputs to their JSON representation
231+
const transformedInput = input.map(arg => {
232+
// Check if the argument is an instance of BamlImage or BamlAudio
233+
// We check the constructor name since the actual classes might be proxied in browser environments
234+
if (arg && typeof arg === 'object' &&
235+
(arg.constructor.name === 'BamlImage' || arg.constructor.name === 'BamlAudio')) {
236+
return arg.toJSON();
237+
}
238+
return arg;
239+
});
240+
241+
response = await action(...transformedInput)
230242

231243
if (isStreamingProps(props) && response instanceof ReadableStream) {
232244
const reader = response.getReader()
@@ -370,12 +382,10 @@ export function use{{ func.name }}(props?: HookInput<'{{ func.name }}', { stream
370382
export function use{{ func.name }}(
371383
props: HookInput<'{{ func.name }}', { stream?: boolean }> = {},
372384
): HookOutput<'{{ func.name }}', { stream: true }> | HookOutput<'{{ func.name }}', { stream: false }> {
373-
if (isNotStreamingProps(props)) {
374-
return useBamlAction(Actions.{{ func.name }}, props)
375-
}
385+
let action = Actions.{{ func.name }};
376386
if (isStreamingProps(props)) {
377-
return useBamlAction(StreamingActions.{{ func.name }}, props)
387+
action = StreamingActions.{{ func.name }};
378388
}
379-
throw new Error('Invalid props')
389+
return useBamlAction(action, props)
380390
}
381391
{%- endfor %}

‎engine/language_client_typescript/artifacts/index.d.ts‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
1-
export { BamlRuntime, FunctionResult, FunctionResultStream, BamlImage as Image, ClientBuilder, BamlAudio as Audio, invoke_runtime_cli, ClientRegistry, BamlLogEvent, } from './native';
1+
export * from './safe_imports';
2+
export * from './errors';
3+
export * from './logging';
4+
export { BamlRuntime, FunctionResult, FunctionResultStream, BamlImage as Image, BamlAudio as Audio, invoke_runtime_cli, ClientRegistry, BamlLogEvent, Collector, FunctionLog, Usage, HTTPRequest, } from './native';
25
export { BamlStream } from './stream';
36
export { BamlCtxManager } from './async_context_vars';
4-
export * from './errors';
57
//# sourceMappingURL=index.d.ts.map

‎engine/language_client_typescript/artifacts/index.d.ts.map‎

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎engine/language_client_typescript/artifacts/index.js‎

Lines changed: 26 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,19 +14,42 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
1414
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
1515
};
1616
Object.defineProperty(exports, "__esModule", { value: true });
17-
exports.BamlCtxManager = exports.BamlStream = exports.BamlLogEvent = exports.ClientRegistry = exports.invoke_runtime_cli = exports.Audio = exports.ClientBuilder = exports.Image = exports.FunctionResultStream = exports.FunctionResult = exports.BamlRuntime = void 0;
17+
exports.BamlCtxManager = exports.BamlStream = exports.HTTPRequest = exports.Usage = exports.FunctionLog = exports.Collector = exports.BamlLogEvent = exports.ClientRegistry = exports.invoke_runtime_cli = exports.Audio = exports.Image = exports.FunctionResultStream = exports.FunctionResult = exports.BamlRuntime = void 0;
18+
__exportStar(require("./safe_imports"), exports);
19+
__exportStar(require("./errors"), exports);
20+
__exportStar(require("./logging"), exports);
21+
// Detect if we're in a Node.js environment
22+
const isNode = typeof process !== 'undefined' &&
23+
process.versions != null &&
24+
process.versions.node != null;
25+
if (!isNode) {
26+
const browserError = (name) => {
27+
throw new Error(`Cannot import ${name} from '@boundaryml/baml' in browser environment. Please import from '@boundaryml/baml/browser' instead.`);
28+
};
29+
// Provide helpful error messages for browser imports
30+
Object.defineProperty(exports, 'Image', {
31+
get: () => browserError('Image'),
32+
enumerable: true,
33+
});
34+
Object.defineProperty(exports, 'Audio', {
35+
get: () => browserError('Audio'),
36+
enumerable: true,
37+
});
38+
}
1839
var native_1 = require("./native");
1940
Object.defineProperty(exports, "BamlRuntime", { enumerable: true, get: function () { return native_1.BamlRuntime; } });
2041
Object.defineProperty(exports, "FunctionResult", { enumerable: true, get: function () { return native_1.FunctionResult; } });
2142
Object.defineProperty(exports, "FunctionResultStream", { enumerable: true, get: function () { return native_1.FunctionResultStream; } });
2243
Object.defineProperty(exports, "Image", { enumerable: true, get: function () { return native_1.BamlImage; } });
23-
Object.defineProperty(exports, "ClientBuilder", { enumerable: true, get: function () { return native_1.ClientBuilder; } });
2444
Object.defineProperty(exports, "Audio", { enumerable: true, get: function () { return native_1.BamlAudio; } });
2545
Object.defineProperty(exports, "invoke_runtime_cli", { enumerable: true, get: function () { return native_1.invoke_runtime_cli; } });
2646
Object.defineProperty(exports, "ClientRegistry", { enumerable: true, get: function () { return native_1.ClientRegistry; } });
2747
Object.defineProperty(exports, "BamlLogEvent", { enumerable: true, get: function () { return native_1.BamlLogEvent; } });
48+
Object.defineProperty(exports, "Collector", { enumerable: true, get: function () { return native_1.Collector; } });
49+
Object.defineProperty(exports, "FunctionLog", { enumerable: true, get: function () { return native_1.FunctionLog; } });
50+
Object.defineProperty(exports, "Usage", { enumerable: true, get: function () { return native_1.Usage; } });
51+
Object.defineProperty(exports, "HTTPRequest", { enumerable: true, get: function () { return native_1.HTTPRequest; } });
2852
var stream_1 = require("./stream");
2953
Object.defineProperty(exports, "BamlStream", { enumerable: true, get: function () { return stream_1.BamlStream; } });
3054
var async_context_vars_1 = require("./async_context_vars");
3155
Object.defineProperty(exports, "BamlCtxManager", { enumerable: true, get: function () { return async_context_vars_1.BamlCtxManager; } });
32-
__exportStar(require("./errors"), exports);

‎engine/language_client_typescript/artifacts/native.d.ts‎

Lines changed: 114 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -23,10 +23,13 @@ export declare class BamlRuntime {
2323
static fromFiles(rootPath: string, files: Record<string, string>, envVars: Record<string, string | undefined | null>): BamlRuntime
2424
reset(rootPath: string, files: Record<string, string>, envVars: Record<string, string>): void
2525
createContextManager(): RuntimeContextManager
26-
callFunction(functionName: string, args: { [name: string]: any }, ctx: RuntimeContextManager, tb?: TypeBuilder | undefined | null, cb?: ClientRegistry | undefined | null): Promise<FunctionResult>
27-
callFunctionSync(functionName: string, args: { [name: string]: any }, ctx: RuntimeContextManager, tb?: TypeBuilder | undefined | null, cb?: ClientRegistry | undefined | null): FunctionResult
28-
streamFunction(functionName: string, args: { [name: string]: any }, cb: ((err: any, param: FunctionResult) => void) | undefined, ctx: RuntimeContextManager, tb?: TypeBuilder | undefined | null, clientRegistry?: ClientRegistry | undefined | null): FunctionResultStream
29-
streamFunctionSync(functionName: string, args: { [name: string]: any }, cb: ((err: any, param: FunctionResult) => void) | undefined, ctx: RuntimeContextManager, tb?: TypeBuilder | undefined | null, clientRegistry?: ClientRegistry | undefined | null): FunctionResultStream
26+
callFunction(functionName: string, args: { [name: string]: any }, ctx: RuntimeContextManager, tb: TypeBuilder | undefined | null, cb: ClientRegistry | undefined | null, collectors: Array<Collector>): Promise<FunctionResult>
27+
callFunctionSync(functionName: string, args: { [name: string]: any }, ctx: RuntimeContextManager, tb: TypeBuilder | undefined | null, cb: ClientRegistry | undefined | null, collectors: Array<Collector>): FunctionResult
28+
streamFunction(functionName: string, args: { [name: string]: any }, cb: ((err: any, param: FunctionResult) => void) | undefined, ctx: RuntimeContextManager, tb: TypeBuilder | undefined | null, clientRegistry: ClientRegistry | undefined | null, collectors: Array<Collector>): FunctionResultStream
29+
streamFunctionSync(functionName: string, args: { [name: string]: any }, cb: ((err: any, param: FunctionResult) => void) | undefined, ctx: RuntimeContextManager, tb: TypeBuilder | undefined | null, clientRegistry: ClientRegistry | undefined | null, collectors: Array<Collector>): FunctionResultStream
30+
buildRequest(functionName: string, args: { [name: string]: any }, ctx: RuntimeContextManager, tb: TypeBuilder | undefined | null, cb: ClientRegistry | undefined | null, stream: boolean): Promise<HTTPRequest>
31+
buildRequestSync(functionName: string, args: { [name: string]: any }, ctx: RuntimeContextManager, tb: TypeBuilder | undefined | null, cb: ClientRegistry | undefined | null, stream: boolean): HTTPRequest
32+
parseLlmResponse(functionName: string, llmResponse: string, allowPartials: boolean, ctx: RuntimeContextManager, tb?: TypeBuilder | undefined | null, cb?: ClientRegistry | undefined | null): any
3033
setLogEventCallback(func?: undefined | ((err: any, param: BamlLogEvent) => void)): void
3134
flush(): void
3235
drainStats(): TraceStats
@@ -54,6 +57,17 @@ export declare class ClientRegistry {
5457
setPrimary(primary: string): void
5558
}
5659

60+
export declare class Collector {
61+
constructor(name?: string | undefined | null)
62+
get logs(): Array<FunctionLog>
63+
get last(): FunctionLog | null
64+
id(functionLogId: string): FunctionLog | null
65+
get usage(): Usage
66+
toString(): string
67+
static __functionSpanCount(): number
68+
static __printStorage(): void
69+
}
70+
5771
export declare class EnumBuilder {
5872
value(name: string): EnumValueBuilder
5973
alias(alias?: string | undefined | null): EnumBuilder
@@ -71,22 +85,100 @@ export declare class FieldType {
7185
optional(): FieldType
7286
}
7387

88+
export declare class FunctionLog {
89+
toString(): string
90+
get id(): string
91+
get functionName(): string
92+
get logType(): string
93+
get timing(): Timing
94+
get usage(): Usage
95+
get calls(): (LLMCall | LLMStreamCall)[]
96+
get rawLlmResponse(): string | null
97+
get selectedCall(): unknown
98+
}
99+
74100
export declare class FunctionResult {
75101
isOk(): boolean
76102
parsed(allowPartials: boolean): any
77103
}
78104

79105
export declare class FunctionResultStream {
80-
onEvent(func: (err: any, param: FunctionResult) => void): void
106+
onEvent(func?: ((err: any, param: FunctionResult) => void) | undefined): void
81107
done(rctx: RuntimeContextManager): Promise<FunctionResult>
82108
}
83109

110+
export declare class HttpBody {
111+
raw(): ArrayBuffer
112+
text(): string
113+
json(): any
114+
}
115+
export type HTTPBody = HttpBody
116+
117+
export declare class HttpRequest {
118+
get id(): string
119+
get body(): HttpBody
120+
toString(): string
121+
get url(): string
122+
get method(): string
123+
get headers(): object
124+
}
125+
export type HTTPRequest = HttpRequest
126+
127+
export declare class HttpResponse {
128+
toString(): string
129+
get status(): number
130+
get headers(): object
131+
get body(): any
132+
}
133+
export type HTTPResponse = HttpResponse
134+
135+
export declare class LlmCall {
136+
get selected(): boolean
137+
get httpRequest(): HTTPRequest | null
138+
get httpResponse(): HTTPResponse | null
139+
get usage(): Usage | null
140+
get timing(): Timing
141+
get provider(): string
142+
get clientName(): string
143+
toString(): string
144+
toString(): string
145+
}
146+
export type LLMCall = LlmCall
147+
148+
export declare class LlmStreamCall {
149+
toString(): string
150+
get httpRequest(): HTTPRequest | null
151+
get httpResponse(): HTTPResponse | null
152+
get provider(): string
153+
get clientName(): string
154+
get selected(): boolean
155+
get usage(): Usage | null
156+
get timing(): StreamTiming
157+
toString(): string
158+
}
159+
export type LLMStreamCall = LlmStreamCall
160+
84161
export declare class RuntimeContextManager {
85162
upsertTags(tags: any): void
86163
deepClone(): RuntimeContextManager
87164
contextDepth(): number
88165
}
89166

167+
export declare class StreamTiming {
168+
toString(): string
169+
get startTimeUtcMs(): number
170+
get durationMs(): number | null
171+
get timeToFirstParsedMs(): number | null
172+
get timeToFirstTokenMs(): number | null
173+
}
174+
175+
export declare class Timing {
176+
toString(): string
177+
get startTimeUtcMs(): number
178+
get durationMs(): number | null
179+
get timeToFirstParsedMs(): number | null
180+
}
181+
90182
export declare class TraceStats {
91183
get failed(): number
92184
get started(): number
@@ -117,6 +209,12 @@ export declare class TypeBuilder {
117209
toString(): string
118210
}
119211

212+
export declare class Usage {
213+
toString(): string
214+
get inputTokens(): number | null
215+
get outputTokens(): number | null
216+
}
217+
120218
export interface BamlLogEvent {
121219
metadata: LogEventMetadata
122220
prompt?: string
@@ -125,11 +223,21 @@ export interface BamlLogEvent {
125223
startTime: string
126224
}
127225

128-
export declare export declare function invoke_runtime_cli(params: Array<string>): void
226+
export declare export declare function get_version(): string
227+
228+
export declare export declare function getLogLevel(): string
229+
230+
export declare export declare function invoke_runtime_cli(params: Array<string>): number
129231

130232
export interface LogEventMetadata {
131233
eventId: string
132234
parentId?: string
133235
rootEventId: string
134236
}
135237

238+
export declare export declare function setLogJsonMode(useJson: boolean): void
239+
240+
export declare export declare function setLogLevel(level: string): void
241+
242+
export declare export declare function setLogMaxChunkLength(length: number): void
243+

‎engine/language_client_typescript/artifacts/native.js‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -368,12 +368,32 @@ module.exports.BamlSpan = nativeBinding.BamlSpan
368368
module.exports.ClassBuilder = nativeBinding.ClassBuilder
369369
module.exports.ClassPropertyBuilder = nativeBinding.ClassPropertyBuilder
370370
module.exports.ClientRegistry = nativeBinding.ClientRegistry
371+
module.exports.Collector = nativeBinding.Collector
371372
module.exports.EnumBuilder = nativeBinding.EnumBuilder
372373
module.exports.EnumValueBuilder = nativeBinding.EnumValueBuilder
373374
module.exports.FieldType = nativeBinding.FieldType
375+
module.exports.FunctionLog = nativeBinding.FunctionLog
374376
module.exports.FunctionResult = nativeBinding.FunctionResult
375377
module.exports.FunctionResultStream = nativeBinding.FunctionResultStream
378+
module.exports.HttpBody = nativeBinding.HttpBody
379+
module.exports.HTTPBody = nativeBinding.HTTPBody
380+
module.exports.HttpRequest = nativeBinding.HttpRequest
381+
module.exports.HTTPRequest = nativeBinding.HTTPRequest
382+
module.exports.HttpResponse = nativeBinding.HttpResponse
383+
module.exports.HTTPResponse = nativeBinding.HTTPResponse
384+
module.exports.LlmCall = nativeBinding.LlmCall
385+
module.exports.LLMCall = nativeBinding.LLMCall
386+
module.exports.LlmStreamCall = nativeBinding.LlmStreamCall
387+
module.exports.LLMStreamCall = nativeBinding.LLMStreamCall
376388
module.exports.RuntimeContextManager = nativeBinding.RuntimeContextManager
389+
module.exports.StreamTiming = nativeBinding.StreamTiming
390+
module.exports.Timing = nativeBinding.Timing
377391
module.exports.TraceStats = nativeBinding.TraceStats
378392
module.exports.TypeBuilder = nativeBinding.TypeBuilder
393+
module.exports.Usage = nativeBinding.Usage
394+
module.exports.get_version = nativeBinding.get_version
395+
module.exports.getLogLevel = nativeBinding.getLogLevel
379396
module.exports.invoke_runtime_cli = nativeBinding.invoke_runtime_cli
397+
module.exports.setLogJsonMode = nativeBinding.setLogJsonMode
398+
module.exports.setLogLevel = nativeBinding.setLogLevel
399+
module.exports.setLogMaxChunkLength = nativeBinding.setLogMaxChunkLength

‎engine/language_client_typescript/artifacts/stream.d.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import { FunctionResultStream, RuntimeContextManager } from './native';
1+
import type { FunctionResultStream, RuntimeContextManager } from './native';
22
export declare class BamlStream<PartialOutputType, FinalOutputType> {
33
private ffiStream;
44
private partialCoerce;

‎engine/language_client_typescript/artifacts/stream.d.ts.map‎

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎engine/language_client_typescript/artifacts/stream.js‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
"use strict";
22
Object.defineProperty(exports, "__esModule", { value: true });
33
exports.BamlStream = void 0;
4-
const _1 = require(".");
4+
const errors_1 = require("./errors");
55
class BamlStream {
66
ffiStream;
77
partialCoerce;
@@ -30,6 +30,7 @@ class BamlStream {
3030
}
3131
finally {
3232
this.eventQueue.push(null);
33+
this.ffiStream.onEvent(undefined);
3334
}
3435
}
3536
driveToCompletionInBg() {
@@ -83,7 +84,7 @@ class BamlStream {
8384
return;
8485
}
8586
catch (err) {
86-
const bamlError = (0, _1.toBamlError)(err instanceof Error ? err : new Error(String(err)));
87+
const bamlError = (0, errors_1.toBamlError)(err instanceof Error ? err : new Error(String(err)));
8788
controller.enqueue(encoder.encode(JSON.stringify({ error: bamlError })));
8889
controller.close();
8990
return;
@@ -92,7 +93,9 @@ class BamlStream {
9293
catch (streamErr) {
9394
const errorPayload = {
9495
type: 'StreamError',
95-
message: streamErr instanceof Error ? streamErr.message : 'Error in stream processing',
96+
message: streamErr instanceof Error
97+
? streamErr.message
98+
: 'Error in stream processing',
9699
prompt: '',
97100
raw_output: '',
98101
};

0 commit comments

Comments
 (0)