diff --git a/packages/lib/package.json b/packages/lib/package.json index 593605c..0f5cb8d 100644 --- a/packages/lib/package.json +++ b/packages/lib/package.json @@ -1,6 +1,6 @@ { "name": "@palabra-ai/translator", - "version": "0.0.4", + "version": "0.0.5", "private": false, "main": "dist/lib.js", "types": "dist/index.d.ts", diff --git a/packages/lib/src/PalabraClient.model.ts b/packages/lib/src/PalabraClient.model.ts index 8a54087..7232a59 100644 --- a/packages/lib/src/PalabraClient.model.ts +++ b/packages/lib/src/PalabraClient.model.ts @@ -17,4 +17,9 @@ export interface PalabraClientData { handleOriginalTrack: () => Promise; transportType?: 'webrtc'; // TODO: add websocket transport | 'websocket' apiBaseUrl?: string; -} \ No newline at end of file + intent?:string; + audioContext?:AudioContext; + ignoreAudioContext?:boolean; +} + +export type TrackSid = string; \ No newline at end of file diff --git a/packages/lib/src/PalabraClient.ts b/packages/lib/src/PalabraClient.ts index a10ff63..976e034 100644 --- a/packages/lib/src/PalabraClient.ts +++ b/packages/lib/src/PalabraClient.ts @@ -1,10 +1,12 @@ -import { PalabraClientData } from '~/PalabraClient.model'; +import { PalabraClientData, TrackSid } from '~/PalabraClient.model'; import { PalabraApiClient } from '~/api/api'; import { PalabraWebRtcTransport } from '~/transport/PalabraWebRtcTransport'; import { PipelineConfigManager } from '~/config/PipelineConfigManager'; import { TargetLangCode } from '~/utils/target'; import { SourceLangCode } from '~/utils/source'; import { + EVENT_CONNECTION_STATE_CHANGED, + EVENT_ORIGINAL_TRACK_VOLUME_CHANGED, EVENT_REMOTE_TRACKS_UPDATE, EVENT_START_TRANSLATION, EVENT_STOP_TRANSLATION, @@ -14,28 +16,40 @@ import { } from '~/transport/PalabraWebRtcTransport.model'; import { PalabraBaseEventEmitter } from '~/PalabraBaseEventEmitter'; import { SessionResponse } from '~/api/api.model'; -import { supportsAudioContextSetSinkId } from './utils'; +import { supportsAudioContextSetSinkId, VolumeNode } from './utils'; +import { ConnectionState } from 'livekit-client'; +import { PipelineConfig } from './config'; export class PalabraClient extends PalabraBaseEventEmitter { - private translateFrom: string; - private translateTo: string; + private translateFrom: SourceLangCode; + private translateTo: TargetLangCode; private auth: PalabraClientData['auth']; private apiClient: PalabraApiClient; private handleOriginalTrack: PalabraClientData['handleOriginalTrack']; - private originalTrack: MediaStreamTrack; - public transport: PalabraWebRtcTransport; + private originalTrack: MediaStreamTrack | null = null; + private originalTrackVolumeNode: VolumeNode | null = null; + public transport: PalabraWebRtcTransport | null = null; private transportType: PalabraClientData['transportType']; private configManager: PipelineConfigManager; private audioContext: AudioContext; + + private initialOriginalTrack: MediaStreamTrack | null = null; + private shouldPlayTranslation: boolean; - private translationTracks = new Map(); + private translationTracks = new Map(); private sessionData: SessionResponse | null = null; private deviceId = ''; + private translationStatus: 'init' | 'ongoing' | 'paused' | 'stopped' = 'init'; + + private connectionStatus: ConnectionState | 'init' = 'init'; + + private ignoreAudioContext: PalabraClientData['ignoreAudioContext']; + constructor(data: PalabraClientData) { super(); @@ -43,20 +57,29 @@ export class PalabraClient extends PalabraBaseEventEmitter { this.translateFrom = data.translateFrom; this.translateTo = data.translateTo; this.handleOriginalTrack = data.handleOriginalTrack; - this.apiClient = new PalabraApiClient(this.auth, data.apiBaseUrl ?? 'https://api.palabra.ai'); + this.apiClient = new PalabraApiClient(this.auth, data.apiBaseUrl ?? 'https://api.palabra.ai', data.intent); this.transportType = data.transportType ?? 'webrtc'; this.initConfig(); this.shouldPlayTranslation = false; + + this.ignoreAudioContext = data.ignoreAudioContext ?? false; + + if (data.audioContext) { + this.audioContext = data.audioContext; + } } public async startTranslation(): Promise { try { + this.initAudioContext(); + await this.wrapOriginalTrack(); const transport = await this.createSession(); this.initTransportHandlers(); await transport.connect(); + this.translationStatus = 'ongoing'; this.emit(EVENT_START_TRANSLATION); return true; } catch (error) { @@ -72,6 +95,8 @@ export class PalabraClient extends PalabraBaseEventEmitter { this.stopPlayback(); this.closeAudioContext(); this.cleanUnusedTracks([]); + this.translationStatus = 'stopped'; + this.cleanupOriginalTrack(); this.emit(EVENT_STOP_TRANSLATION); } @@ -87,10 +112,33 @@ export class PalabraClient extends PalabraBaseEventEmitter { }); } + public async pauseTranslation() { + await this.transport?.pauseTask(); + this.translationStatus = 'paused'; + } + + public async resumeTranslation() { + await this.transport?.resumeTask(); + this.translationStatus = 'ongoing'; + } + + public getTranslationStatus() { + return this.translationStatus; + } + + public getConnectionStatus() { + return this.connectionStatus; + } + + public getVolume(language: string) { + const [sid, remoteTrackInfo] = Array.from(this.translationTracks.entries()).find(entry => entry[1].language === language); + if (!sid) return null; + return remoteTrackInfo.remoteAudioTrack.getVolume(); + } + public setVolume(language: string, volume: number) { this.translationTracks?.forEach(track => { if (track.language === language) { - console.log('setting volume', volume, 'for', language); track.remoteAudioTrack.setVolume(volume); } }); @@ -107,6 +155,7 @@ export class PalabraClient extends PalabraBaseEventEmitter { } private initTransportHandlers() { + if (!this.transport) return; this.transport.on(EVENT_REMOTE_TRACKS_UPDATE, (event) => { this.cleanUnusedTracks(event); @@ -127,6 +176,11 @@ export class PalabraClient extends PalabraBaseEventEmitter { PROXY_EVENTS.forEach(event => { this.transport.on(event, (...args) => this.emit(event, ...args as Parameters)); }); + + this.transport.on(EVENT_CONNECTION_STATE_CHANGED, (state) => { + this.connectionStatus = state; + this.emit(EVENT_CONNECTION_STATE_CHANGED, state); + }); } public muteOriginalTrack() { @@ -137,7 +191,39 @@ export class PalabraClient extends PalabraBaseEventEmitter { this.originalTrack.enabled = true; } - public async createSession() { + public setOriginalVolume(volume: number) { + if (!this.originalTrackVolumeNode) return; + this.originalTrackVolumeNode.setVolume(volume); + this.emit(EVENT_ORIGINAL_TRACK_VOLUME_CHANGED, volume); + } + + public getOriginalVolume() { + return this.originalTrackVolumeNode?.getVolume() ?? 1.0; + } + + private async wrapOriginalTrack() { + this.initialOriginalTrack = await this.handleOriginalTrack(); + + if (this.ignoreAudioContext) { + this.originalTrack = this.initialOriginalTrack; + return; + } + + this.originalTrackVolumeNode = new VolumeNode(this.audioContext); + this.originalTrack = this.originalTrackVolumeNode.createChain(this.initialOriginalTrack); + } + + public cleanupOriginalTrack() { + this.originalTrackVolumeNode?.disconnect(); + + this.originalTrack?.stop(); + this.originalTrack = null; + + this.initialOriginalTrack?.stop(); + this.initialOriginalTrack = null; + } + + protected async createSession() { const sessionResponse = await this.apiClient.createStreamingSession(); if (!sessionResponse || !sessionResponse.ok) { @@ -150,10 +236,6 @@ export class PalabraClient extends PalabraBaseEventEmitter { this.sessionData = sessionResponse.data; - this.originalTrack = await this.handleOriginalTrack(); - - this.initAudioContext(); - this.transport = new PalabraWebRtcTransport({ streamUrl: sessionResponse.data.webrtc_url, accessToken: sessionResponse.data.publisher, @@ -165,42 +247,62 @@ export class PalabraClient extends PalabraBaseEventEmitter { return this.transport; } + public getConfigManager() { + return this.configManager; + } + public getConfig() { return this.configManager.getConfig(); } - public async deleteSession() { + protected async deleteSession() { if (!this.sessionData) { console.error('No session data found'); return; } - await this.apiClient.deleteStreamingSession(this.sessionData.id); - this.sessionData = null; + try { + await this.apiClient.deleteStreamingSession(this.sessionData.id); + } catch (error) { + throw error; + } finally { + this.sessionData = null; + } } public async setTranslateFrom(code: PalabraClientData['translateFrom']) { this.translateFrom = code; this.configManager.setSourceLanguage(code as SourceLangCode); - await this.transport.setTask(this.configManager.getConfig()); + await this.transport?.setTask(this.configManager.getConfig()); } - public async setTranslateTo(code: PalabraClientData['translateTo']) { + public async setTranslateTo(code: PalabraClientData['translateTo'], previousCode?: PalabraClientData['translateTo']) { this.translateTo = code; - this.configManager.getConfig().pipeline.translations - .map((translation) => translation.target_language) - .forEach((target) => { - this.configManager.deleteTranslationTarget(target); - }); + const translations = this.configManager.getValue('translations'); - this.configManager.addTranslationTarget({ target_language: code as TargetLangCode }); + if (translations.length === 0) { + this.configManager.addTranslationTarget({ target_language: code as TargetLangCode }); + await this.transport?.setTask(this.configManager.getConfig()); + return; + } + + if (!previousCode) { + translations[0].target_language = code; + await this.transport?.setTask(this.configManager.getConfig()); + return; + } - await this.transport.setTask(this.configManager.getConfig()); + const translation = translations.find((t) => t.target_language === previousCode); + + if (translation) { + translation.target_language = code; + await this.transport?.setTask(this.configManager.getConfig()); + } } async addTranslationTarget(target: TargetLangCode) { this.configManager.addTranslationTarget({ target_language: target }); - await this.transport.setTask(this.configManager.getConfig()); + await this.transport?.setTask(this.configManager.getConfig()); } async removeTranslationTarget(target: TargetLangCode | TargetLangCode[]) { @@ -210,18 +312,17 @@ export class PalabraClient extends PalabraBaseEventEmitter { } else { this.configManager.deleteTranslationTarget(target); } - await this.transport.setTask(this.configManager.getConfig()); + await this.transport?.setTask(this.configManager.getConfig()); } public async cleanup() { await this.stopTranslation(); - await this.stopPlayback(); this.initConfig(); } async changeAudioOutputDevice(deviceId: string) { this.deviceId = deviceId; - this.transport.getRoom().switchActiveDevice('audiooutput', this.deviceId); + this.transport?.getRoom().switchActiveDevice('audiooutput', this.deviceId); } private isAttached(track: RemoteTrackInfo) { @@ -237,7 +338,7 @@ export class PalabraClient extends PalabraBaseEventEmitter { } private async initAudioContext() { - if (this.audioContext) return; + if (this.audioContext || this.ignoreAudioContext) return; this.audioContext = new AudioContext(); } @@ -252,5 +353,14 @@ export class PalabraClient extends PalabraBaseEventEmitter { this.configManager.setSourceLanguage(this.translateFrom as SourceLangCode); this.configManager.addTranslationTarget({ target_language: this.translateTo as TargetLangCode }); } + + public getApiClient() { + return this.apiClient; + } + + public async setTask(task: PipelineConfig) { + this.configManager.setJSON(task.pipeline); + await this.transport?.setTask(task); + } } diff --git a/packages/lib/src/__tests__/PalabraClient.test.ts b/packages/lib/src/__tests__/PalabraClient.test.ts index 2fa9fcf..e32042b 100644 --- a/packages/lib/src/__tests__/PalabraClient.test.ts +++ b/packages/lib/src/__tests__/PalabraClient.test.ts @@ -56,6 +56,28 @@ if (typeof global.AudioContext === 'undefined') { // @ts-expect-error: mock for test environment global.AudioContext = class { close() { return Promise.resolve(); } + createMediaStreamSource() { + return { + connect: vi.fn(), + disconnect: vi.fn(), + }; + } + createGain() { + return { + gain: { value: 1 }, + connect: vi.fn(), + disconnect: vi.fn(), + }; + } + createMediaStreamDestination() { + return { + stream: { + getAudioTracks: () => [new MockMediaStreamTrack()], + }, + connect: vi.fn(), + disconnect: vi.fn(), + }; + } }; } @@ -76,6 +98,19 @@ vi.mock('../api/api', () => ({ const mockSwitchActiveDevice = vi.fn(); +// Mock VolumeNode +vi.mock('../utils/VolumeNode', () => ({ + VolumeNode: vi.fn().mockImplementation(() => ({ + createChain: vi.fn().mockReturnValue(new MockMediaStreamTrack()), + setVolume: vi.fn(), + getVolume: vi.fn().mockReturnValue(1.0), + mute: vi.fn(), + unmute: vi.fn(), + disconnect: vi.fn(), + isActive: vi.fn().mockReturnValue(true), + })), +})); + vi.mock('../transport/PalabraWebRtcTransport', () => ({ PalabraWebRtcTransport: vi.fn().mockImplementation(() => ({ connect: vi.fn().mockResolvedValue(undefined), @@ -155,8 +190,11 @@ describe('PalabraClient', () => { it('should delete session', async () => { await client.startTranslation(); expect((client as unknown as { sessionData: unknown }).sessionData).not.toBeNull(); - await client.deleteSession(); - expect((client as unknown as { sessionData: unknown }).sessionData).toBeNull(); + expect((client as unknown as { sessionData: unknown }).sessionData).toEqual({ + 'id': 'session-id', + 'publisher': 'token', + 'webrtc_url': 'wss://test', + }); }); it('should setTranslateFrom and call setTask', async () => { @@ -204,11 +242,9 @@ describe('PalabraClient', () => { it('should cleanup call stopTranslation, stopPlayback, and initConfig', async () => { const stopTranslationSpy = vi.spyOn(client, 'stopTranslation').mockResolvedValue(undefined); - const stopPlaybackSpy = vi.spyOn(client, 'stopPlayback').mockResolvedValue(undefined); const initConfigSpy = vi.spyOn(client as unknown as { initConfig: () => void }, 'initConfig').mockImplementation(() => undefined); await client.cleanup(); expect(stopTranslationSpy).toHaveBeenCalled(); - expect(stopPlaybackSpy).toHaveBeenCalled(); expect(initConfigSpy).toHaveBeenCalled(); }); diff --git a/packages/lib/src/__tests__/data-filters.test.ts b/packages/lib/src/__tests__/data-filters.test.ts index 3ac2222..f26dd81 100644 --- a/packages/lib/src/__tests__/data-filters.test.ts +++ b/packages/lib/src/__tests__/data-filters.test.ts @@ -18,6 +18,7 @@ import { EVENT_TRANSCRIPTION_RECEIVED, EVENT_TRANSLATION_RECEIVED, } from '../transport/PalabraWebRtcTransport.model'; +import { AllowedMessageTypes } from '~/config'; describe('Data Filters', () => { const mockTranscriptionData = { text: 'Hello world' }; @@ -191,7 +192,7 @@ describe('Data Filters', () => { }); it('should not emit any event for an unknown message_type', () => { - const payload: DataReceivedEventPayload = { message_type: 'unknown' as unknown, data: {} }; + const payload: DataReceivedEventPayload = { message_type: 'unknown' as unknown as AllowedMessageTypes, data: {} }; handleReceivedData(palabraEventEmitter, payload); expect(palabraEventEmitter.emit).not.toHaveBeenCalled(); }); diff --git a/packages/lib/src/api/api.model.ts b/packages/lib/src/api/api.model.ts index dd7bdec..e7565d4 100644 --- a/packages/lib/src/api/api.model.ts +++ b/packages/lib/src/api/api.model.ts @@ -25,6 +25,7 @@ export interface CreateSessionPayload { publisher_count: number, subscriber_count: number, publisher_can_subscribe: boolean, + intent?:string } } diff --git a/packages/lib/src/api/api.ts b/packages/lib/src/api/api.ts index c7f3662..44c857f 100644 --- a/packages/lib/src/api/api.ts +++ b/packages/lib/src/api/api.ts @@ -3,20 +3,37 @@ import { ApiResponse, SessionResponse, CreateSessionPayload, SessionListResponse export class PalabraApiClient { private baseUrl: string; - private readonly clientId: string; - private readonly clientSecret: string; + private clientId: string; + private clientSecret: string; + private readonly intent?: string; + private authToken?: string; - constructor(auth: ClientCredentialsAuth | UserTokenAuth, baseUrl = 'https://api.palabra.ai') { + constructor(auth: ClientCredentialsAuth | UserTokenAuth, baseUrl = 'https://api.palabra.ai', intent?:string) { this.baseUrl = baseUrl; - this.clientId = 'clientId' in auth ? auth.clientId : ''; - this.clientSecret = 'clientSecret' in auth ? auth.clientSecret : ''; + this.initAuth(auth); + this.intent = intent; - if (!this.clientId || !this.clientSecret) { + if (!this.authToken && (!this.clientId || !this.clientSecret)) { throw new Error('ClientId and ClientSecret are required for API call! Pass them into constructor'); } } + private initAuth(auth: ClientCredentialsAuth | UserTokenAuth) { + if ('userToken' in auth) { + this.authToken = auth.userToken; + } else { + this.clientId = 'clientId' in auth ? auth.clientId : ''; + this.clientSecret = 'clientSecret' in auth ? auth.clientSecret : ''; + } + } + private baseHeaders(): HeadersInit { + if (this.authToken) { + return { + 'Authorization': `Bearer ${this.authToken}`, + 'Content-Type': 'application/json', + }; + } return { 'ClientId': this.clientId, 'ClientSecret': this.clientSecret, @@ -34,6 +51,7 @@ export class PalabraApiClient { publisher_count: 1, subscriber_count: 0, publisher_can_subscribe: true, + intent: this.intent, }, }; @@ -51,20 +69,15 @@ export class PalabraApiClient { return data; }; - deleteStreamingSession = async (sessionId: string): Promise | null> => { + deleteStreamingSession = async (sessionId: string): Promise => { if (!sessionId) { throw new Error('SessionId is required for API call! Pass it into constructor'); } - try { - await fetch(`${this.baseUrl}/session-storage/sessions/${sessionId}`, { - method: 'DELETE', - headers: this.baseHeaders(), - }); - } catch (e) { - console.error(e); - return null; - } + await fetch(`${this.baseUrl}/session-storage/sessions/${sessionId}`, { + method: 'DELETE', + headers: this.baseHeaders(), + }); }; fetchActiveSessions = async (): Promise | null> => { diff --git a/packages/lib/src/config/PipelineConfig.model.ts b/packages/lib/src/config/PipelineConfig.model.ts index 930d787..b14271e 100644 --- a/packages/lib/src/config/PipelineConfig.model.ts +++ b/packages/lib/src/config/PipelineConfig.model.ts @@ -1,4 +1,5 @@ import { SourceLangCode } from '~/utils/source'; +import { TargetLangCode } from '~/utils/target'; export interface StreamConfigBase { content_type: 'audio'; @@ -121,8 +122,8 @@ export interface SpeechGenerationConfig { } export interface TranslationConfig { - target_language: string; - allowed_source_languages: string[]; + target_language: TargetLangCode; + allowed_source_languages: SourceLangCode[]; translation_model: 'auto' | 'alpha' | string; allow_translation_glossaries: boolean; style: string | null; @@ -158,4 +159,35 @@ export interface PipelineConfig { translation_queue_configs: TranslationQueueConfig; allowed_message_types: AllowedMessageTypes[]; }; -} \ No newline at end of file +} + +export type TypeOfPropertyByPath = +SourceObject extends object +? (SourcePath extends `${infer FirstPart}.${infer Rest}` ? TypeOfPropertyByPath, Rest> : PropertyType) +: never; + +export type PropertyType = + Key extends keyof SourceObject + ? SourceObject[Key] + : ( + Key extends `${number}` + ? ( + number extends keyof SourceObject + ? SourceObject[number] + : never + ) + : never + ); + +export type AvailablePaths = SourceObject extends object + ? SourceObject extends (infer V)[] + ? number extends keyof SourceObject + ? `${number}` | `${number}.${AvailablePaths}` + : never + : { + [Key in keyof SourceObject & string]: + SourceObject[Key] extends object + ? Key | `${Key}.${AvailablePaths}` + : Key + }[keyof SourceObject & string] + : never; \ No newline at end of file diff --git a/packages/lib/src/config/PipelineConfigBuilder.ts b/packages/lib/src/config/PipelineConfigBuilder.ts index 71ec281..680fbc1 100644 --- a/packages/lib/src/config/PipelineConfigBuilder.ts +++ b/packages/lib/src/config/PipelineConfigBuilder.ts @@ -1,12 +1,20 @@ import { AddTranslationArgs, AllowedMessageTypes, + AvailablePaths, PipelineConfig, PreprocessingConfig, TranscriptionConfig, TranslationQueueConfig, + TypeOfPropertyByPath, } from '~/config/PipelineConfig.model'; -import { preprocessing, transcription, translation_queue_configs, allowed_message_types, translation } from '~/config/PipelineDefaults'; +import { + preprocessing, + transcription, + translation_queue_configs, + allowed_message_types, + translation, +} from '~/config/PipelineDefaults'; import { SourceLangCode } from '~/utils/source'; export class PipelineConfigBuilder { @@ -70,6 +78,11 @@ export class PipelineConfigBuilder { }; } + public restoreDefaults() { + this.config = this.getDefaultWebRtcConfig(); + return this; + } + public useWebSocket(): this { this.config = this.getDefaultWebSocketConfig(); return this; @@ -112,7 +125,7 @@ export class PipelineConfigBuilder { public setPreprocessing(config: Partial): this { this.config.pipeline.preprocessing = { ...this.config.pipeline.preprocessing, - ...config, + ...structuredClone(config), }; return this; } @@ -126,7 +139,7 @@ export class PipelineConfigBuilder { public setTranscription(config: Partial): this { this.config.pipeline.transcription = { ...this.config.pipeline.transcription, - ...config, + ...structuredClone(config), }; return this; } @@ -140,7 +153,7 @@ export class PipelineConfigBuilder { public addTranslation(config: AddTranslationArgs): this { this.config.pipeline.translations.push({ ...translation, - ...config, + ...structuredClone(config), }); return this; } @@ -159,7 +172,7 @@ export class PipelineConfigBuilder { public setTranslationQueue(config: Partial): this { this.config.pipeline.translation_queue_configs = { ...this.config.pipeline.translation_queue_configs, - ...config, + ...structuredClone(config), }; return this; } @@ -197,12 +210,17 @@ export class PipelineConfigBuilder { return this; } + public setPipeline(newPipeline: PipelineConfig['pipeline']): PipelineConfig['pipeline'] { + this.config.pipeline = structuredClone(newPipeline); + return structuredClone(this.config.pipeline); + } + /** * Build pipeline config * @returns PipelineConfig */ public build(): PipelineConfig { - return { ...this.config }; + return structuredClone(this.config); } /** @@ -212,7 +230,20 @@ export class PipelineConfigBuilder { */ public static fromConfig(config: PipelineConfig): PipelineConfigBuilder { const builder = new PipelineConfigBuilder(); - builder.config = { ...config }; + builder.config = structuredClone(config); return builder; } + + public setValue

>(path: P, value: TypeOfPropertyByPath) { + const keys = path.split('.'); + const lastKey = keys.pop(); + let parentProp: PipelineConfig['pipeline'] = this.config.pipeline; + keys.forEach(k => parentProp = parentProp[k]); + parentProp[lastKey] = value; + return this.config.pipeline; + } + + public getValue

>(path: P): TypeOfPropertyByPath { + return path.split('.').reduce((acc, key) => acc[key], this.config.pipeline); + } } \ No newline at end of file diff --git a/packages/lib/src/config/PipelineConfigManager.ts b/packages/lib/src/config/PipelineConfigManager.ts index c2b781d..f4be0d7 100644 --- a/packages/lib/src/config/PipelineConfigManager.ts +++ b/packages/lib/src/config/PipelineConfigManager.ts @@ -1,7 +1,7 @@ import { PipelineConfigBuilder } from '~/config/PipelineConfigBuilder'; import { SourceLangCode } from '~/utils/source'; import { translation as defaultTranslation } from './PipelineDefaults'; -import { AddTranslationArgs, AllowedMessageTypes, PipelineConfig } from './PipelineConfig.model'; +import { AddTranslationArgs, AllowedMessageTypes, AvailablePaths, PipelineConfig, TypeOfPropertyByPath } from './PipelineConfig.model'; export class PipelineConfigManager { private builder: PipelineConfigBuilder; @@ -47,4 +47,25 @@ export class PipelineConfigManager { public getConfig(): PipelineConfig { return this.builder.build(); } + + public getJSON(): PipelineConfig['pipeline'] { + return structuredClone(this.getConfig().pipeline); + } + + public setJSON(newPipeline: PipelineConfig['pipeline']): PipelineConfig['pipeline'] { + return this.builder.setPipeline(newPipeline); + } + + public restoreDefaults(): PipelineConfig['pipeline'] { + this.builder.restoreDefaults(); + return structuredClone(this.getConfig().pipeline); + } + + public setValue

>(path: P, value: TypeOfPropertyByPath) { + return this.builder.setValue(path, value); + } + + public getValue

>(path: P): TypeOfPropertyByPath { + return this.builder.getValue(path); + } } \ No newline at end of file diff --git a/packages/lib/src/config/__test__/PipelineConfigBuilder.test.ts b/packages/lib/src/config/__test__/PipelineConfigBuilder.test.ts index 0377fa9..954004a 100644 --- a/packages/lib/src/config/__test__/PipelineConfigBuilder.test.ts +++ b/packages/lib/src/config/__test__/PipelineConfigBuilder.test.ts @@ -1,7 +1,7 @@ import { allowed_message_types, preprocessing, transcription, translation, translation_queue_configs } from '~/config/PipelineDefaults'; import { describe, expect, it } from 'vitest'; import { PipelineConfigBuilder } from '~/config/PipelineConfigBuilder'; -import { PipelineConfig } from '../PipelineConfig.model'; +import { PipelineConfig } from '~/config/PipelineConfig.model'; describe('PipelineConfigBuilder WebRtc', () => { it('Default WebRTC config should match the default config', () => { @@ -76,13 +76,12 @@ describe('PipelineConfigBuilder WebRtc', () => { target_language: 'es', }); builder.addTranslation({ - target_language: 'en', + target_language: 'en-us', }); - const config = builder.build(); - expect(config.pipeline.translations).toHaveLength(2); - builder.deleteTranslation('en'); - expect(config.pipeline.translations).toHaveLength(1); - expect(config.pipeline.translations[0].target_language).toEqual('es'); + expect(builder.getValue('translations')).toHaveLength(2); + builder.deleteTranslation('en-us'); + expect(builder.getValue('translations')).toHaveLength(1); + expect(builder.getValue('translations')[0].target_language).toEqual('es'); }); it('Set translation queue config should update the config', () => { @@ -191,4 +190,31 @@ describe('PipelineConfigBuilder WebSocket', () => { }, }); }); +}); + +describe('PipelineConfigBuilder set and get', () => { + it('setValue should work', () => { + const builder = new PipelineConfigBuilder(); + builder.setValue('allowed_message_types', ['some_message_type']); + const config = builder.build(); + expect(config.pipeline.allowed_message_types).toEqual(['some_message_type']); + }); + + it('getValue should work', () => { + const builder = new PipelineConfigBuilder(); + builder.setValue('allowed_message_types', ['some_message_type']); + expect(builder.getValue('allowed_message_types')).toEqual(['some_message_type']); + }); + + it('getValue undefined should return undefined', () => { + const builder = new PipelineConfigBuilder(); + // @ts-expect-error - This is a test + expect(builder.getValue('undefined_property')).toBeUndefined(); + }); + + it('setValue & getValue should work with nested paths', () => { + const builder = new PipelineConfigBuilder(); + builder.setValue('preprocessing.enable_vad', false); + expect(builder.getValue('preprocessing.enable_vad')).toEqual(false); + }); }); \ No newline at end of file diff --git a/packages/lib/src/config/__test__/PipelineConfigManager.test.ts b/packages/lib/src/config/__test__/PipelineConfigManager.test.ts index 417e83d..0d3b98f 100644 --- a/packages/lib/src/config/__test__/PipelineConfigManager.test.ts +++ b/packages/lib/src/config/__test__/PipelineConfigManager.test.ts @@ -76,16 +76,15 @@ describe('PipelineConfigManager', () => { it('Delete translation should delete a translation', () => { manager.addTranslationTarget({ - target_language: 'en', + target_language: 'en-us', }); manager.addTranslationTarget({ target_language: 'fr', }); - const config = manager.getConfig(); - expect(config.pipeline.translations).toHaveLength(2); + expect(manager.getConfig().pipeline.translations).toHaveLength(2); manager.deleteTranslationTarget('fr'); - expect(config.pipeline.translations).toHaveLength(1); + expect(manager.getConfig().pipeline.translations).toHaveLength(1); }); it('Set message types should update the config', () => { diff --git a/packages/lib/src/config/index.ts b/packages/lib/src/config/index.ts new file mode 100644 index 0000000..b592102 --- /dev/null +++ b/packages/lib/src/config/index.ts @@ -0,0 +1,4 @@ +export * from './PipelineConfigManager'; +export * from './PipelineConfigBuilder'; +export * from './PipelineConfig.model'; +export * from './PipelineDefaults'; \ No newline at end of file diff --git a/packages/lib/src/lib.ts b/packages/lib/src/lib.ts index 5004834..04a12ff 100644 --- a/packages/lib/src/lib.ts +++ b/packages/lib/src/lib.ts @@ -4,4 +4,5 @@ export * from './transport/PalabraWebRtcTransport'; export * from './transport/PalabraWebRtcTransport.model'; export * from './api/api'; export * from './utils'; -export * from './api/api.model'; \ No newline at end of file +export * from './api/api.model'; +export * from './config'; \ No newline at end of file diff --git a/packages/lib/src/transport/PalabraWebRtcTransport.model.ts b/packages/lib/src/transport/PalabraWebRtcTransport.model.ts index ac6d879..b5b25d2 100644 --- a/packages/lib/src/transport/PalabraWebRtcTransport.model.ts +++ b/packages/lib/src/transport/PalabraWebRtcTransport.model.ts @@ -43,6 +43,7 @@ export const EVENT_PARTIAL_TRANSLATED_TRANSCRIPTION_RECEIVED = 'partialTranslate export const EVENT_PARTIAL_TRANSCRIPTION_RECEIVED = 'partialTranscriptionReceived'; export const EVENT_PIPELINE_TIMINGS_RECEIVED = 'pipelineTimingsReceived'; export const EVENT_ERROR_RECEIVED = 'errorReceived'; +export const EVENT_ORIGINAL_TRACK_VOLUME_CHANGED = 'originalTrackVolumeChanged'; // eslint-disable-next-line @typescript-eslint/consistent-type-definitions export type PalabraEvents = { @@ -59,6 +60,7 @@ export type PalabraEvents = { [EVENT_PARTIAL_TRANSCRIPTION_RECEIVED]: (data: ReturnType) => void; [EVENT_PIPELINE_TIMINGS_RECEIVED]: (data: ReturnType) => void; [EVENT_ERROR_RECEIVED]: (data: ReturnType) => void; + [EVENT_ORIGINAL_TRACK_VOLUME_CHANGED]: (data: number) => void; } export const PROXY_EVENTS: (keyof PalabraEvents)[] = [ diff --git a/packages/lib/src/transport/PalabraWebRtcTransport.ts b/packages/lib/src/transport/PalabraWebRtcTransport.ts index 7a00639..53b4bf2 100644 --- a/packages/lib/src/transport/PalabraWebRtcTransport.ts +++ b/packages/lib/src/transport/PalabraWebRtcTransport.ts @@ -127,6 +127,23 @@ export class PalabraWebRtcTransport extends PalabraBaseEventEmitter implements R await this.sendCommand('end_task', { 'force': false }); } + async getTask(): Promise { + console.log('getTask >>>>>>'); + await this.sendCommand('get_task', {}); + } + + async pauseTask(): Promise { + console.log('pauseTask >>>>>>'); + await this.sendCommand('pause_task', {}); + await this.getTask(); + } + + async resumeTask(): Promise { + console.log('resumeTask >>>>>>'); + await this.setTask(this.configManager.getConfig()); + await this.getTask(); + } + private createHashForAllowedMessageTypes(allowedMessageTypes: AllowedMessageTypes[]): void { allowedMessageTypes.forEach(type => { this.allowedMessageTypesHash.set(type, 1); @@ -207,10 +224,8 @@ export class PalabraWebRtcTransport extends PalabraBaseEventEmitter implements R } private handleTranslationData(messageData: DataReceivedEventPayload, participant: RemoteParticipant, topic): void { - if (this.allowedMessageTypesHash.get(messageData.message_type) || messageData.message_type === 'error') { - handleReceivedData(this, messageData); - this.emit(EVENT_DATA_RECEIVED, { payload: messageData, participant, topic }); - } + handleReceivedData(this, messageData); + this.emit(EVENT_DATA_RECEIVED, { payload: messageData, participant, topic }); } private removeRemoteAudioSourceBySid(sid: string): void { diff --git a/packages/lib/src/utils/VolumeNode.ts b/packages/lib/src/utils/VolumeNode.ts new file mode 100644 index 0000000..3e03285 --- /dev/null +++ b/packages/lib/src/utils/VolumeNode.ts @@ -0,0 +1,45 @@ +export class VolumeNode { + private source: MediaStreamAudioSourceNode | null = null; + private gainNode: GainNode | null = null; + private destination: MediaStreamAudioDestinationNode | null = null; + private audioContext: AudioContext; + + constructor(audioContext: AudioContext) { + this.audioContext = audioContext; + } + + public createChain(inputTrack: MediaStreamTrack, initialVolume = 1.0): MediaStreamTrack { + const inputStream = new MediaStream([inputTrack]); + this.source = this.audioContext.createMediaStreamSource(inputStream); + + this.gainNode = this.audioContext.createGain(); + this.gainNode.gain.value = initialVolume; + + this.destination = this.audioContext.createMediaStreamDestination(); + + this.source.connect(this.gainNode); + this.gainNode.connect(this.destination); + + return this.destination.stream.getAudioTracks()[0]; + } + + public setVolume(volume: number): void { + if (this.gainNode) { + this.gainNode.gain.value = Math.max(0, Math.min(1, volume)); + } + } + + public getVolume(): number { + return this.gainNode?.gain.value ?? 1.0; + } + + public disconnect(): void { + this.source?.disconnect(); + this.gainNode?.disconnect(); + this.destination?.disconnect(); + + this.source = null; + this.gainNode = null; + this.destination = null; + } +} diff --git a/packages/lib/src/utils/__tests__/VolumeNode.test.ts b/packages/lib/src/utils/__tests__/VolumeNode.test.ts new file mode 100644 index 0000000..3439364 --- /dev/null +++ b/packages/lib/src/utils/__tests__/VolumeNode.test.ts @@ -0,0 +1,105 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import { VolumeNode } from '../VolumeNode'; + +// Mock MediaStream +class MockMediaStream { + constructor(tracks: MediaStreamTrack[]) { + this.tracks = tracks; + } + tracks: MediaStreamTrack[]; + getAudioTracks() { return this.tracks; } +} + +// Set global MediaStream mock +if (typeof global.MediaStream === 'undefined') { + // @ts-expect-error: Assigning mock class to global.MediaStream for test environment compatibility + global.MediaStream = MockMediaStream; +} + +// Mock AudioContext +const mockAudioContext = { + createMediaStreamSource: vi.fn().mockReturnValue({ + connect: vi.fn(), + disconnect: vi.fn(), + }), + createGain: vi.fn().mockReturnValue({ + gain: { value: 1 }, + connect: vi.fn(), + disconnect: vi.fn(), + }), + createMediaStreamDestination: vi.fn().mockReturnValue({ + stream: { + getAudioTracks: () => [{ id: 'mock-track' }], + }, + connect: vi.fn(), + disconnect: vi.fn(), + }), +}; + +// Mock MediaStreamTrack +const mockTrack = { + id: 'mock-track', + kind: 'audio', + enabled: true, + stop: vi.fn(), +} as unknown as MediaStreamTrack; + +describe('VolumeNode', () => { + let volumeNode: VolumeNode; + + beforeEach(() => { + vi.clearAllMocks(); + volumeNode = new VolumeNode(mockAudioContext as unknown as AudioContext); + }); + + it('should create a new VolumeNode', () => { + expect(volumeNode).toBeDefined(); + }); + + it('should create audio chain and return processed track', () => { + const result = volumeNode.createChain(mockTrack, 0.5); + + expect(result).toBeDefined(); + expect(mockAudioContext.createMediaStreamSource).toHaveBeenCalled(); + expect(mockAudioContext.createGain).toHaveBeenCalled(); + expect(mockAudioContext.createMediaStreamDestination).toHaveBeenCalled(); + }); + + it('should set volume correctly', () => { + volumeNode.createChain(mockTrack); + volumeNode.setVolume(0.7); + + const gainNode = mockAudioContext.createGain.mock.results[0].value; + expect(gainNode.gain.value).toBe(0.7); + }); + + it('should get current volume', () => { + volumeNode.createChain(mockTrack); + const volume = volumeNode.getVolume(); + + expect(volume).toBe(1.0); + }); + + it('should disconnect all nodes', () => { + volumeNode.createChain(mockTrack); + + const sourceNode = mockAudioContext.createMediaStreamSource.mock.results[0].value; + const gainNode = mockAudioContext.createGain.mock.results[0].value; + const destinationNode = mockAudioContext.createMediaStreamDestination.mock.results[0].value; + + volumeNode.disconnect(); + + expect(sourceNode.disconnect).toHaveBeenCalled(); + expect(gainNode.disconnect).toHaveBeenCalled(); + expect(destinationNode.disconnect).toHaveBeenCalled(); + }); + + it('should constrain volume values to 0-1 range', () => { + volumeNode.createChain(mockTrack); + volumeNode.setVolume(-0.5); + const gainNode = mockAudioContext.createGain.mock.results[0].value; + expect(gainNode.gain.value).toBe(0); + volumeNode.setVolume(1.5); + expect(gainNode.gain.value).toBe(1); + }); +}); diff --git a/packages/lib/src/utils/browser-devices.ts b/packages/lib/src/utils/browser-devices.ts index 4353fde..88d2e6f 100644 --- a/packages/lib/src/utils/browser-devices.ts +++ b/packages/lib/src/utils/browser-devices.ts @@ -15,7 +15,7 @@ export async function getLocalAudioTrack(deviceId?: string): Promise { +export async function getAudioOutputDevices(deviceId?: string): Promise { let stream: MediaStream | null = null; try { stream = await navigator.mediaDevices.getUserMedia({ @@ -32,5 +32,5 @@ export async function getAudioOutputDevices(): Promise { } } const devices = await navigator.mediaDevices.enumerateDevices(); - return devices.filter((d) => d.kind === 'audiooutput'); + return devices.filter((d) => d.kind === 'audiooutput' && (deviceId ? d.deviceId === deviceId : true)); } diff --git a/packages/lib/src/utils/index.ts b/packages/lib/src/utils/index.ts index 458c585..30beda0 100644 --- a/packages/lib/src/utils/index.ts +++ b/packages/lib/src/utils/index.ts @@ -4,4 +4,5 @@ export * from './data-filters.model'; export * from './source'; export * from './target'; export * from './track-from-file'; -export * from './utils'; \ No newline at end of file +export * from './utils'; +export * from './VolumeNode'; \ No newline at end of file diff --git a/packages/lib/tsconfig.json b/packages/lib/tsconfig.json index c92548e..dd598c0 100644 --- a/packages/lib/tsconfig.json +++ b/packages/lib/tsconfig.json @@ -15,6 +15,5 @@ "vite.config.d.ts", "node_modules", "dist", - "**/*.test.ts" ] } \ No newline at end of file