From 9966804ba1ef61f84a324b44a8c554c7d668e48b Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Wed, 23 Apr 2025 18:08:05 -0500 Subject: [PATCH 1/8] init --- .../common/audio-player/AudioPlayer.svelte | 14 +- src/lib/helpers/pcmProcessor.js | 56 ++++++ src/lib/services/llm-realtime-service.js | 1 - src/lib/services/realtime-chat-service.js | 181 ++++++++++++++++++ .../[conversationId]/chat-box.svelte | 11 +- 5 files changed, 251 insertions(+), 12 deletions(-) create mode 100644 src/lib/helpers/pcmProcessor.js create mode 100644 src/lib/services/realtime-chat-service.js diff --git a/src/lib/common/audio-player/AudioPlayer.svelte b/src/lib/common/audio-player/AudioPlayer.svelte index 6de707bb..5e5f730e 100644 --- a/src/lib/common/audio-player/AudioPlayer.svelte +++ b/src/lib/common/audio-player/AudioPlayer.svelte @@ -242,7 +242,7 @@ if (loop === "none") { if (order === "list") { if ($playList.playingIndex < audios.length - 1) { - const promise = buildNextSongPromise(nextIdx); + const promise = buildNextAudioPromise(nextIdx); promise.then(() => play()); } else { $playList.playingIndex = ($playList.playingIndex + 1) % audios.length; @@ -257,13 +257,13 @@ } else { targetIdx = randomIdx; } - const promise = buildNextSongPromise(targetIdx); + const promise = buildNextAudioPromise(targetIdx); promise.then(() => play()); } } else if (loop === "one") { player.currentTime = 0; } else if (loop === "all") { - const promise = buildNextSongPromise(nextIdx); + const promise = buildNextAudioPromise(nextIdx); promise.then(() => play()); } }; @@ -271,7 +271,7 @@ /** * @param {number} idx */ - function buildNextSongPromise(idx) { + function buildNextAudioPromise(idx) { return new Promise((/** @type {any} */ resolve) => { $playList.playingIndex = idx; player.currentTime = 0; @@ -287,8 +287,8 @@ /** * @param {number} idx */ - function switchSong(idx) { - const promise = buildNextSongPromise(idx); + function switchAudio(idx) { + const promise = buildNextAudioPromise(idx); if (autoPlayNextOnClick) { promise.then(() => { play(); @@ -490,7 +490,7 @@ {#each $audioList as song, idx} -
  • switchSong(idx) }> +
  • switchAudio(idx) }> {#if idx === $playList.playingIndex} {/if} diff --git a/src/lib/helpers/pcmProcessor.js b/src/lib/helpers/pcmProcessor.js new file mode 100644 index 00000000..250644b7 --- /dev/null +++ b/src/lib/helpers/pcmProcessor.js @@ -0,0 +1,56 @@ + +export const AudioRecordingWorklet = ` +class AudioProcessingWorklet extends AudioWorkletProcessor { + + // send and clear buffer every 2048 samples, + // which at 16khz is about 8 times a second + buffer = new Int16Array(2048); + + // current write index + bufferWriteIndex = 0; + + constructor() { + super(); + this.hasAudio = false; + } + + /** + * @param inputs Float32Array[][] [input#][channel#][sample#] so to access first inputs 1st channel inputs[0][0] + * @param outputs Float32Array[][] + */ + process(inputs) { + if (inputs[0].length) { + const channel0 = inputs[0][0]; + this.processChunk(channel0); + } + return true; + } + + sendAndClearBuffer(){ + this.port.postMessage({ + event: "chunk", + data: { + int16arrayBuffer: this.buffer.slice(0, this.bufferWriteIndex).buffer, + }, + }); + this.bufferWriteIndex = 0; + } + + processChunk(float32Array) { + const l = float32Array.length; + + for (let i = 0; i < l; i++) { + // convert float32 -1 to 1 to int16 -32768 to 32767 + const int16Value = float32Array[i] * 32768; + this.buffer[this.bufferWriteIndex++] = int16Value; + if(this.bufferWriteIndex >= this.buffer.length) { + this.sendAndClearBuffer(); + } + } + + if(this.bufferWriteIndex >= this.buffer.length) { + this.sendAndClearBuffer(); + } + } +} +`; \ No newline at end of file diff --git a/src/lib/services/llm-realtime-service.js b/src/lib/services/llm-realtime-service.js index e90f2e6a..f5121575 100644 --- a/src/lib/services/llm-realtime-service.js +++ b/src/lib/services/llm-realtime-service.js @@ -1,7 +1,6 @@ import { endpoints } from '$lib/services/api-endpoints.js'; import { replaceUrl } from '$lib/helpers/http'; import axios from 'axios'; -import { json } from '@sveltejs/kit'; export const llmRealtime = { /** @type {RTCPeerConnection} */ diff --git a/src/lib/services/realtime-chat-service.js b/src/lib/services/realtime-chat-service.js new file mode 100644 index 00000000..8b77b868 --- /dev/null +++ b/src/lib/services/realtime-chat-service.js @@ -0,0 +1,181 @@ +import { AudioRecordingWorklet } from "$lib/helpers/pcmProcessor"; + +// @ts-ignore +const SpeechRecognition = window.SpeechRecognition || window.webkitSpeechRecognition; + +// @ts-ignore +const AudioContext = window.AudioContext || window.webkitAudioContext; + +export const realtimeChat = { + + /** @type {WebSocket | null} */ + socket: null, + + /** @type {MediaRecorder | null} */ + mediaRecorder: null, + + /** @type {MediaStream | null} */ + mediaStream: null, + + /** @type {SpeechRecognition | null} */ + recognition: null, + + /** + * @param {string} agentId + * @param {string} conversationId + */ + start(agentId, conversationId) { + this.socket = new WebSocket(`ws://localhost:5100/chat/stream/${agentId}/${conversationId}`); + + this.socket.onopen = async () => { + console.log("WebSocket connected"); + + this.socket?.send(JSON.stringify({ + event: "start" + })); + + this.mediaStream = await navigator.mediaDevices.getUserMedia({ audio: true }); + const audioCtx = new AudioContext({ sampleRate: 16000 }); + + const workletName = "audio-recorder-worklet"; + const src = createWorketFromSrc(workletName, AudioRecordingWorklet); + await audioCtx.audioWorklet.addModule(src); + + const workletNode = new AudioWorkletNode(audioCtx, workletName); + const micSource = audioCtx.createMediaStreamSource(this.mediaStream); + micSource.connect(workletNode); + + workletNode.port.onmessage = event => { + const arrayBuffer = event.data.data.int16arrayBuffer; + if (arrayBuffer && this.socket?.readyState === WebSocket.OPEN) { + const arrayBufferString = arrayBufferToBase64(arrayBuffer); + this.socket.send(JSON.stringify({ + event: 'media', + payload: arrayBufferString + })); + } + }; + + // this.recognition = new SpeechRecognition(); + // this.recognition.continuous = true; + // this.recognition.interimResults = false; + // this.recognition.lang = "en-US"; + + // this.recognition.onresult = (/** @type { any } */ event) => { + // const lastResult = event.results[event.results.length - 1]; + // const transcript = lastResult[0].transcript.trim(); + + // console.log("Recognized:", transcript); + + // const message = { + // event: "media", + // payload: transcript + // }; + + // if (this.socket?.readyState === WebSocket.OPEN) { + // this.socket.send(JSON.stringify(message)); + // } + // }; + + // this.recognition.onend = () => { + // console.log('Speech recognition closed.'); + // }; + // this.recognition.start(); + + // navigator.mediaDevices.getUserMedia({ audio: true }) + // .then(stream => { + // this.mediaStream = stream; + // this.mediaRecorder = new MediaRecorder(stream, { mimeType: "audio/webm" }); + // /** @type {any[]} */ + // let audioChunks = []; + // this.mediaRecorder.ondataavailable = (/** @type {any} */ event) => { + // if (event.data.size > 0) { + // // audioChunks.push(event.data); + // } + // }; + + // this.mediaRecorder.onstop = async () => { + // console.log('mediaRecorder stopped'); + // // const blob = new Blob(audioChunks, { type: 'audio/webm' }); + // // const arrayBuffer = await blob.arrayBuffer(); + + // // // Decode audio and downsample to PCM16 + // // const audioCtx = new AudioContext({ sampleRate: 16000 }); + // // const audioBuffer = await audioCtx.decodeAudioData(arrayBuffer); + + // // const channelData = audioBuffer.getChannelData(0); // mono + // // const pcm16 = new Int16Array(channelData.length); + + // // for (let i = 0; i < channelData.length; i++) { + // // pcm16[i] = Math.max(-1, Math.min(1, channelData[i])) * 32767; + // // } + + // // const pcmBytes = new Uint8Array(pcm16.buffer); + // // const base64 = btoa(String.fromCharCode(...pcmBytes)); + // // console.log(base64); + // }; + + // this.mediaRecorder.start(); + // }) + // .catch((err) => { + // console.error("Failed to access microphone", err); + // }); + }; + + this.socket.onclose = () => { + console.log("Websocket closed"); + } + + this.socket.onerror = (/** @type {any} */ e) => console.error('WebSocket error', e); + }, + + stop() { + if (this.mediaRecorder) { + this.mediaRecorder.stop(); + } + + if (this.mediaStream) { + this.mediaStream.getTracks().forEach(t => t.stop()); + this.mediaStream = null; + } + + if (this.recognition) { + this.recognition.stop(); + } + + if (this.socket?.readyState === WebSocket.OPEN) { + this.socket.send(JSON.stringify({ + event: 'disconnect' + })); + this.socket.close(); + } + } +}; + +/** + * @param {ArrayBuffer} buffer + */ +function arrayBufferToBase64(buffer) { + var binary = ""; + var bytes = new Uint8Array(buffer); + var len = bytes.byteLength; + for (var i = 0; i < len; i++) { + binary += String.fromCharCode(bytes[i]); + } + return window.btoa(binary); +} + +/** + * @param {string} workletName + * @param {string} workletSrc + */ +function createWorketFromSrc(workletName, workletSrc) { + const script = new Blob( + [`registerProcessor("${workletName}", ${workletSrc})`], + { + type: "application/javascript", + }, + ); + + return URL.createObjectURL(script); +}; \ No newline at end of file diff --git a/src/routes/chat/[agentId]/[conversationId]/chat-box.svelte b/src/routes/chat/[agentId]/[conversationId]/chat-box.svelte index ced8e358..6744949c 100644 --- a/src/routes/chat/[agentId]/[conversationId]/chat-box.svelte +++ b/src/routes/chat/[agentId]/[conversationId]/chat-box.svelte @@ -71,6 +71,7 @@ import PersistLog from './persist-log/persist-log.svelte'; import InstantLog from './instant-log/instant-log.svelte'; import LocalStorageManager from '$lib/helpers/utils/storage-manager'; + import { realtimeChat } from '$lib/services/realtime-chat-service'; const options = { @@ -673,13 +674,15 @@ if (disableSpeech) return; if (!isListening) { - llmRealtime.start(params.agentId, (/** @type {any} */ message) => { - console.log(message); - }); + // llmRealtime.start(params.agentId, (/** @type {any} */ message) => { + // console.log(message); + // }); + realtimeChat.start(params.agentId, params.conversationId); isListening = true; microphoneIcon = "microphone"; } else { - llmRealtime.stop(); + // llmRealtime.stop(); + realtimeChat.stop(); isListening = false; microphoneIcon = "microphone-off"; } From 8248b0518a6c4ba95618ac649ddd0a3ff1a60904 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Thu, 24 Apr 2025 17:53:11 -0500 Subject: [PATCH 2/8] add audio output --- src/lib/helpers/pcmProcessor.js | 7 +- src/lib/services/realtime-chat-service.js | 241 +++++++++++------- .../[conversationId]/chat-box.svelte | 5 - 3 files changed, 149 insertions(+), 104 deletions(-) diff --git a/src/lib/helpers/pcmProcessor.js b/src/lib/helpers/pcmProcessor.js index 250644b7..a0dcfbb1 100644 --- a/src/lib/helpers/pcmProcessor.js +++ b/src/lib/helpers/pcmProcessor.js @@ -3,7 +3,8 @@ export const AudioRecordingWorklet = ` class AudioProcessingWorklet extends AudioWorkletProcessor { // send and clear buffer every 2048 samples, - // which at 16khz is about 8 times a second + // which at 16khz is about 8 times a second, + // or at 24khz is about 11 times a second buffer = new Int16Array(2048); // current write index @@ -43,12 +44,12 @@ class AudioProcessingWorklet extends AudioWorkletProcessor { // convert float32 -1 to 1 to int16 -32768 to 32767 const int16Value = float32Array[i] * 32768; this.buffer[this.bufferWriteIndex++] = int16Value; - if(this.bufferWriteIndex >= this.buffer.length) { + if (this.bufferWriteIndex >= this.buffer.length) { this.sendAndClearBuffer(); } } - if(this.bufferWriteIndex >= this.buffer.length) { + if (this.bufferWriteIndex >= this.buffer.length) { this.sendAndClearBuffer(); } } diff --git a/src/lib/services/realtime-chat-service.js b/src/lib/services/realtime-chat-service.js index 8b77b868..9245bf8b 100644 --- a/src/lib/services/realtime-chat-service.js +++ b/src/lib/services/realtime-chat-service.js @@ -1,31 +1,42 @@ +import { PUBLIC_SERVICE_URL } from "$env/static/public"; import { AudioRecordingWorklet } from "$lib/helpers/pcmProcessor"; -// @ts-ignore -const SpeechRecognition = window.SpeechRecognition || window.webkitSpeechRecognition; - // @ts-ignore const AudioContext = window.AudioContext || window.webkitAudioContext; +const sampleRate = 24000; + +/** @type {AudioContext} */ +let audioCtx = new AudioContext(); + +/** @type {any[]} */ +let audioQueue = []; + +/** @type {boolean} */ +let isPlaying = false; + export const realtimeChat = { /** @type {WebSocket | null} */ socket: null, - /** @type {MediaRecorder | null} */ - mediaRecorder: null, - /** @type {MediaStream | null} */ mediaStream: null, - /** @type {SpeechRecognition | null} */ - recognition: null, + /** @type {AudioWorkletNode | null} */ + workletNode: null, + + /** @type {MediaStreamAudioSourceNode | null} */ + micSource: null, /** * @param {string} agentId * @param {string} conversationId */ start(agentId, conversationId) { - this.socket = new WebSocket(`ws://localhost:5100/chat/stream/${agentId}/${conversationId}`); + reset(); + const wsUrl = buildWebsocketUrl(); + this.socket = new WebSocket(`${wsUrl}/chat/stream/${agentId}/${conversationId}`); this.socket.onopen = async () => { console.log("WebSocket connected"); @@ -35,17 +46,17 @@ export const realtimeChat = { })); this.mediaStream = await navigator.mediaDevices.getUserMedia({ audio: true }); - const audioCtx = new AudioContext({ sampleRate: 16000 }); + audioCtx = new AudioContext({ sampleRate: sampleRate }); const workletName = "audio-recorder-worklet"; - const src = createWorketFromSrc(workletName, AudioRecordingWorklet); + const src = createWorkletFromSrc(workletName, AudioRecordingWorklet); await audioCtx.audioWorklet.addModule(src); - const workletNode = new AudioWorkletNode(audioCtx, workletName); - const micSource = audioCtx.createMediaStreamSource(this.mediaStream); - micSource.connect(workletNode); + this.workletNode = new AudioWorkletNode(audioCtx, workletName); + this.micSource = audioCtx.createMediaStreamSource(this.mediaStream); + this.micSource.connect(this.workletNode); - workletNode.port.onmessage = event => { + this.workletNode.port.onmessage = event => { const arrayBuffer = event.data.data.int16arrayBuffer; if (arrayBuffer && this.socket?.readyState === WebSocket.OPEN) { const arrayBufferString = arrayBufferToBase64(arrayBuffer); @@ -55,92 +66,41 @@ export const realtimeChat = { })); } }; + }; - // this.recognition = new SpeechRecognition(); - // this.recognition.continuous = true; - // this.recognition.interimResults = false; - // this.recognition.lang = "en-US"; - - // this.recognition.onresult = (/** @type { any } */ event) => { - // const lastResult = event.results[event.results.length - 1]; - // const transcript = lastResult[0].transcript.trim(); - - // console.log("Recognized:", transcript); - - // const message = { - // event: "media", - // payload: transcript - // }; - - // if (this.socket?.readyState === WebSocket.OPEN) { - // this.socket.send(JSON.stringify(message)); - // } - // }; - - // this.recognition.onend = () => { - // console.log('Speech recognition closed.'); - // }; - // this.recognition.start(); - - // navigator.mediaDevices.getUserMedia({ audio: true }) - // .then(stream => { - // this.mediaStream = stream; - // this.mediaRecorder = new MediaRecorder(stream, { mimeType: "audio/webm" }); - // /** @type {any[]} */ - // let audioChunks = []; - // this.mediaRecorder.ondataavailable = (/** @type {any} */ event) => { - // if (event.data.size > 0) { - // // audioChunks.push(event.data); - // } - // }; - - // this.mediaRecorder.onstop = async () => { - // console.log('mediaRecorder stopped'); - // // const blob = new Blob(audioChunks, { type: 'audio/webm' }); - // // const arrayBuffer = await blob.arrayBuffer(); - - // // // Decode audio and downsample to PCM16 - // // const audioCtx = new AudioContext({ sampleRate: 16000 }); - // // const audioBuffer = await audioCtx.decodeAudioData(arrayBuffer); - - // // const channelData = audioBuffer.getChannelData(0); // mono - // // const pcm16 = new Int16Array(channelData.length); - - // // for (let i = 0; i < channelData.length; i++) { - // // pcm16[i] = Math.max(-1, Math.min(1, channelData[i])) * 32767; - // // } - - // // const pcmBytes = new Uint8Array(pcm16.buffer); - // // const base64 = btoa(String.fromCharCode(...pcmBytes)); - // // console.log(base64); - // }; - - // this.mediaRecorder.start(); - // }) - // .catch((err) => { - // console.error("Failed to access microphone", err); - // }); + this.socket.onmessage = (/** @type {MessageEvent} */ e) => { + try { + const json = JSON.parse(e.data); + if (json.event === 'media' && !!json.media.payload) { + const data = json.media.payload; + enqueueAudioChunk(data); + } + } catch { + // console.error('Error when parsing message'); + } }; this.socket.onclose = () => { console.log("Websocket closed"); - } + }; - this.socket.onerror = (/** @type {any} */ e) => console.error('WebSocket error', e); + this.socket.onerror = (/** @type {Event} */ e) => { + console.error('WebSocket error', e); + }; }, stop() { - if (this.mediaRecorder) { - this.mediaRecorder.stop(); - } - + reset(); + if (this.mediaStream) { this.mediaStream.getTracks().forEach(t => t.stop()); this.mediaStream = null; } - if (this.recognition) { - this.recognition.stop(); + if (this.workletNode) { + this.micSource?.disconnect(this.workletNode); + this.workletNode.port.close(); + this.workletNode.disconnect(); } if (this.socket?.readyState === WebSocket.OPEN) { @@ -152,24 +112,66 @@ export const realtimeChat = { } }; + +function buildWebsocketUrl() { + let url = ''; + const host = PUBLIC_SERVICE_URL.split('://'); + + if (PUBLIC_SERVICE_URL.startsWith('https')) { + url = `wss:${host[1]}`; + } else if (PUBLIC_SERVICE_URL.startsWith('http')) { + url = `ws:${host[1]}`; + } + + return url; +} + +function reset() { + isPlaying = false; + audioQueue = []; +} + /** - * @param {ArrayBuffer} buffer + * @param {string} base64Audio */ -function arrayBufferToBase64(buffer) { - var binary = ""; - var bytes = new Uint8Array(buffer); - var len = bytes.byteLength; - for (var i = 0; i < len; i++) { - binary += String.fromCharCode(bytes[i]); +function enqueueAudioChunk(base64Audio) { + const arrayBuffer = base64ToArrayBuffer(base64Audio); + const float32Data = convert16BitPCMToFloat32(arrayBuffer); + + const audioBuffer = audioCtx.createBuffer(1, float32Data.length, sampleRate); + audioBuffer.getChannelData(0).set(float32Data); + audioQueue.push(audioBuffer); + + if (!isPlaying) { + playNext(); } - return window.btoa(binary); } +function playNext() { + if (audioQueue.length === 0) { + isPlaying = false; + return; + } + + isPlaying = true; + const buffer = audioQueue.shift(); + + const source = audioCtx.createBufferSource(); + source.buffer = buffer; + source.connect(audioCtx.destination); + + source.onended = () => { + playNext(); + }; + source.start(); +} + + /** * @param {string} workletName * @param {string} workletSrc */ -function createWorketFromSrc(workletName, workletSrc) { +function createWorkletFromSrc(workletName, workletSrc) { const script = new Blob( [`registerProcessor("${workletName}", ${workletSrc})`], { @@ -178,4 +180,51 @@ function createWorketFromSrc(workletName, workletSrc) { ); return URL.createObjectURL(script); +}; + + +/** + * @param {ArrayBuffer} buffer + */ +function arrayBufferToBase64(buffer) { + var binary = ""; + var bytes = new Uint8Array(buffer); + var len = bytes.byteLength; + for (var i = 0; i < len; i++) { + binary += String.fromCharCode(bytes[i]); + } + return btoa(binary); +}; + +/** + * @param {string} base64 + */ +function base64ToArrayBuffer(base64) { + const binaryStr = atob(base64); + const len = binaryStr.length; + const bytes = new Uint8Array(len); + + for (let i = 0; i < len; i++) { + bytes[i] = binaryStr.charCodeAt(i); + } + return bytes.buffer; +}; + +/** + * @param {ArrayBuffer} buffer + */ +function convert16BitPCMToFloat32(buffer) { + const chunk = new Uint8Array(buffer); + const output = new Float32Array(chunk.length / 2); + const dataView = new DataView(chunk.buffer); + + for (let i = 0; i< chunk.length / 2; i++) { + try { + const int16 = dataView.getInt16(i * 2, true); + output[i] = int16 / 32767; + } catch (e) { + console.error(e); + } + } + return output; }; \ No newline at end of file diff --git a/src/routes/chat/[agentId]/[conversationId]/chat-box.svelte b/src/routes/chat/[agentId]/[conversationId]/chat-box.svelte index 6744949c..25f99160 100644 --- a/src/routes/chat/[agentId]/[conversationId]/chat-box.svelte +++ b/src/routes/chat/[agentId]/[conversationId]/chat-box.svelte @@ -46,7 +46,6 @@ } from '$env/static/public'; import { BOT_SENDERS, LEARNER_ID, TRAINING_MODE, USER_SENDERS, ADMIN_ROLES, IMAGE_DATA_PREFIX } from '$lib/helpers/constants'; import { signalr } from '$lib/services/signalr-service.js'; - import { llmRealtime } from '$lib/services/llm-realtime-service.js'; import { newConversation } from '$lib/services/conversation-service'; import DialogModal from '$lib/common/DialogModal.svelte'; import HeadTitle from '$lib/common/HeadTitle.svelte'; @@ -674,14 +673,10 @@ if (disableSpeech) return; if (!isListening) { - // llmRealtime.start(params.agentId, (/** @type {any} */ message) => { - // console.log(message); - // }); realtimeChat.start(params.agentId, params.conversationId); isListening = true; microphoneIcon = "microphone"; } else { - // llmRealtime.stop(); realtimeChat.stop(); isListening = false; microphoneIcon = "microphone-off"; From ea0a5a2a56ca4d4c7150862020a42890ae7c3006 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Fri, 25 Apr 2025 13:39:12 -0500 Subject: [PATCH 3/8] refine realtime --- .../helpers/{ => realtime}/pcmProcessor.js | 6 +- src/lib/services/realtime-chat-service.js | 79 ++++++++++--------- 2 files changed, 47 insertions(+), 38 deletions(-) rename src/lib/helpers/{ => realtime}/pcmProcessor.js (89%) diff --git a/src/lib/helpers/pcmProcessor.js b/src/lib/helpers/realtime/pcmProcessor.js similarity index 89% rename from src/lib/helpers/pcmProcessor.js rename to src/lib/helpers/realtime/pcmProcessor.js index a0dcfbb1..948973e8 100644 --- a/src/lib/helpers/pcmProcessor.js +++ b/src/lib/helpers/realtime/pcmProcessor.js @@ -10,9 +10,11 @@ class AudioProcessingWorklet extends AudioWorkletProcessor { // current write index bufferWriteIndex = 0; + speaking = false; + threshold = 0.1; + constructor() { super(); - this.hasAudio = false; } /** @@ -22,6 +24,7 @@ class AudioProcessingWorklet extends AudioWorkletProcessor { process(inputs) { if (inputs[0].length) { const channel0 = inputs[0][0]; + this.speaking = channel0.some(sample => Math.abs(sample) >= this.threshold); this.processChunk(channel0); } return true; @@ -31,6 +34,7 @@ class AudioProcessingWorklet extends AudioWorkletProcessor { this.port.postMessage({ event: "chunk", data: { + speaking: this.speaking, int16arrayBuffer: this.buffer.slice(0, this.bufferWriteIndex).buffer, }, }); diff --git a/src/lib/services/realtime-chat-service.js b/src/lib/services/realtime-chat-service.js index 9245bf8b..fd214850 100644 --- a/src/lib/services/realtime-chat-service.js +++ b/src/lib/services/realtime-chat-service.js @@ -1,5 +1,5 @@ import { PUBLIC_SERVICE_URL } from "$env/static/public"; -import { AudioRecordingWorklet } from "$lib/helpers/pcmProcessor"; +import { AudioRecordingWorklet } from "$lib/helpers/realtime/pcmProcessor"; // @ts-ignore const AudioContext = window.AudioContext || window.webkitAudioContext; @@ -15,20 +15,20 @@ let audioQueue = []; /** @type {boolean} */ let isPlaying = false; -export const realtimeChat = { - - /** @type {WebSocket | null} */ - socket: null, +/** @type {WebSocket | null} */ +let socket = null; - /** @type {MediaStream | null} */ - mediaStream: null, +/** @type {MediaStream | null} */ +let mediaStream = null; - /** @type {AudioWorkletNode | null} */ - workletNode: null, +/** @type {AudioWorkletNode | null} */ +let workletNode = null; - /** @type {MediaStreamAudioSourceNode | null} */ - micSource: null, +/** @type {MediaStreamAudioSourceNode | null} */ +let micSource = null; +export const realtimeChat = { + /** * @param {string} agentId * @param {string} conversationId @@ -36,31 +36,34 @@ export const realtimeChat = { start(agentId, conversationId) { reset(); const wsUrl = buildWebsocketUrl(); - this.socket = new WebSocket(`${wsUrl}/chat/stream/${agentId}/${conversationId}`); + socket = new WebSocket(`${wsUrl}/chat/stream/${agentId}/${conversationId}`); - this.socket.onopen = async () => { + socket.onopen = async () => { console.log("WebSocket connected"); - this.socket?.send(JSON.stringify({ + socket?.send(JSON.stringify({ event: "start" })); - this.mediaStream = await navigator.mediaDevices.getUserMedia({ audio: true }); + mediaStream = await navigator.mediaDevices.getUserMedia({ audio: true }); audioCtx = new AudioContext({ sampleRate: sampleRate }); const workletName = "audio-recorder-worklet"; const src = createWorkletFromSrc(workletName, AudioRecordingWorklet); await audioCtx.audioWorklet.addModule(src); - this.workletNode = new AudioWorkletNode(audioCtx, workletName); - this.micSource = audioCtx.createMediaStreamSource(this.mediaStream); - this.micSource.connect(this.workletNode); + workletNode = new AudioWorkletNode(audioCtx, workletName); + micSource = audioCtx.createMediaStreamSource(mediaStream); + micSource.connect(workletNode); - this.workletNode.port.onmessage = event => { + workletNode.port.onmessage = event => { const arrayBuffer = event.data.data.int16arrayBuffer; - if (arrayBuffer && this.socket?.readyState === WebSocket.OPEN) { + if (arrayBuffer && socket?.readyState === WebSocket.OPEN) { + if (event.data.data.speaking) { + reset(); + } const arrayBufferString = arrayBufferToBase64(arrayBuffer); - this.socket.send(JSON.stringify({ + socket.send(JSON.stringify({ event: 'media', payload: arrayBufferString })); @@ -68,7 +71,7 @@ export const realtimeChat = { }; }; - this.socket.onmessage = (/** @type {MessageEvent} */ e) => { + socket.onmessage = (/** @type {MessageEvent} */ e) => { try { const json = JSON.parse(e.data); if (json.event === 'media' && !!json.media.payload) { @@ -80,11 +83,11 @@ export const realtimeChat = { } }; - this.socket.onclose = () => { + socket.onclose = () => { console.log("Websocket closed"); }; - this.socket.onerror = (/** @type {Event} */ e) => { + socket.onerror = (/** @type {Event} */ e) => { console.error('WebSocket error', e); }; }, @@ -92,22 +95,25 @@ export const realtimeChat = { stop() { reset(); - if (this.mediaStream) { - this.mediaStream.getTracks().forEach(t => t.stop()); - this.mediaStream = null; + if (mediaStream) { + mediaStream.getTracks().forEach(t => t.stop()); + mediaStream = null; } - if (this.workletNode) { - this.micSource?.disconnect(this.workletNode); - this.workletNode.port.close(); - this.workletNode.disconnect(); + if (workletNode) { + micSource?.disconnect(workletNode); + workletNode.port.close(); + workletNode.disconnect(); + micSource = null; + workletNode = null; } - if (this.socket?.readyState === WebSocket.OPEN) { - this.socket.send(JSON.stringify({ + if (socket?.readyState === WebSocket.OPEN) { + socket.send(JSON.stringify({ event: 'disconnect' })); - this.socket.close(); + socket.close(); + socket = null; } } }; @@ -143,7 +149,7 @@ function enqueueAudioChunk(base64Audio) { audioQueue.push(audioBuffer); if (!isPlaying) { - playNext(); + playNext(); } } @@ -159,7 +165,6 @@ function playNext() { const source = audioCtx.createBufferSource(); source.buffer = buffer; source.connect(audioCtx.destination); - source.onended = () => { playNext(); }; @@ -221,7 +226,7 @@ function convert16BitPCMToFloat32(buffer) { for (let i = 0; i< chunk.length / 2; i++) { try { const int16 = dataView.getInt16(i * 2, true); - output[i] = int16 / 32767; + output[i] = int16 / 32768; } catch (e) { console.error(e); } From c38d704ee0eef8ee600ce3f933968356ede12c9f Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Fri, 25 Apr 2025 14:13:00 -0500 Subject: [PATCH 4/8] change data format --- src/lib/services/realtime-chat-service.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/lib/services/realtime-chat-service.js b/src/lib/services/realtime-chat-service.js index fd214850..df26040f 100644 --- a/src/lib/services/realtime-chat-service.js +++ b/src/lib/services/realtime-chat-service.js @@ -65,7 +65,9 @@ export const realtimeChat = { const arrayBufferString = arrayBufferToBase64(arrayBuffer); socket.send(JSON.stringify({ event: 'media', - payload: arrayBufferString + body: { + payload: arrayBufferString + } })); } }; From 5535e5506ab61886f4d9c28161f35bc49c23072a Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Mon, 28 Apr 2025 14:53:02 -0500 Subject: [PATCH 5/8] add agent links --- src/lib/common/ProfileDropdown.svelte | 3 +- src/lib/helpers/types/agentTypes.js | 8 + src/lib/scss/custom/pages/_agent.scss | 2 +- src/routes/page/agent/[agentId]/+page.svelte | 32 +-- .../agent-prompt-wrapper.svelte | 91 +++++++++ .../agent-prompt/agent-link.svelte | 185 ++++++++++++++++++ .../{ => agent-prompt}/agent-template.svelte | 15 +- src/routes/page/user/me/+page.svelte | 3 +- 8 files changed, 312 insertions(+), 27 deletions(-) create mode 100644 src/routes/page/agent/[agentId]/agent-components/agent-prompt-wrapper.svelte create mode 100644 src/routes/page/agent/[agentId]/agent-components/agent-prompt/agent-link.svelte rename src/routes/page/agent/[agentId]/agent-components/{ => agent-prompt}/agent-template.svelte (95%) diff --git a/src/lib/common/ProfileDropdown.svelte b/src/lib/common/ProfileDropdown.svelte index 41d9c802..00fa0681 100644 --- a/src/lib/common/ProfileDropdown.svelte +++ b/src/lib/common/ProfileDropdown.svelte @@ -42,7 +42,8 @@ > handleAvatarLoad(e)} /> diff --git a/src/lib/helpers/types/agentTypes.js b/src/lib/helpers/types/agentTypes.js index aa199b54..ed9701c5 100644 --- a/src/lib/helpers/types/agentTypes.js +++ b/src/lib/helpers/types/agentTypes.js @@ -10,6 +10,13 @@ * @property {string} content */ +/** + * @typedef {Object} AgentLink + * @property {string?} [uid] + * @property {string} name + * @property {string} content + */ + /** * @typedef {Object} AgentLlmConfig * @property {boolean} is_inherit - Inherited from default Agent settings @@ -65,6 +72,7 @@ * @property {import('$pluginTypes').PluginDefModel} plugin * @property {FunctionDef[]} functions * @property {AgentTemplate[]} templates + * @property {AgentLink[]} links * @property {Object[]} responses * @property {RoutingRule[]} routing_rules * @property {AgentRule[]} rules diff --git a/src/lib/scss/custom/pages/_agent.scss b/src/lib/scss/custom/pages/_agent.scss index a16d0ecd..2cddf4b5 100644 --- a/src/lib/scss/custom/pages/_agent.scss +++ b/src/lib/scss/custom/pages/_agent.scss @@ -100,7 +100,7 @@ .agent-prompt-header { background-color: white; - padding: 20px; + padding: 15px; } .agent-prompt-body { diff --git a/src/routes/page/agent/[agentId]/+page.svelte b/src/routes/page/agent/[agentId]/+page.svelte index ddbf636c..e8a812d1 100644 --- a/src/routes/page/agent/[agentId]/+page.svelte +++ b/src/routes/page/agent/[agentId]/+page.svelte @@ -19,7 +19,7 @@ import { goto } from '$app/navigation'; import { AgentExtensions } from '$lib/helpers/utils/agent'; import LocalStorageManager from '$lib/helpers/utils/storage-manager'; - import AgentTemplate from './agent-components/agent-template.svelte'; + import AgentPromptWrapper from './agent-components/agent-prompt-wrapper.svelte'; /** @type {import('$agentTypes').AgentModel} */ let agent; @@ -28,7 +28,7 @@ /** @type {any} */ let agentInstructionCmp = null; /** @type {any} */ - let agentTemplateCmp = null; + let agentPromptWrapperCmp = null; /** @type {any} */ let agentTabsCmp = null; /** @type {import('$agentTypes').AgentModel} */ @@ -94,7 +94,7 @@ function handleAgentUpdate() { fetchJsonContent(); fetchInstructions(); - fetchTemplates(); + fetchPromptWrapper(); fetchTabData(); agent = { @@ -115,7 +115,7 @@ isComplete = true; deleteAgentDraft(); refreshInstructions(); - refreshTemplates(); + refreshPromptWrapper(); setTimeout(() => { isComplete = false; }, duration); @@ -164,20 +164,22 @@ } // Templates - function formatOriginalTemplates() { - const obj = agentTemplateCmp?.fetchOriginalTemplates(); + function formatOriginalPromptWrapper() { + const obj = agentPromptWrapperCmp?.fetchOriginalPromptWrapperData(); return { - templates: obj.templates || [] + templates: obj.templates || [], + links: obj.links || [] } } - function fetchTemplates() { - const obj = agentTemplateCmp?.fetchTemplates(); + function fetchPromptWrapper() { + const obj = agentPromptWrapperCmp?.fetchPromptWrapperData(); agent.templates = obj.templates || []; + agent.links = obj.links || []; } - function refreshTemplates() { - agentTemplateCmp?.refresh(); + function refreshPromptWrapper() { + agentPromptWrapperCmp?.refresh(); } @@ -232,7 +234,7 @@ ...agent, ...formatJsonContent(), ...formatOriginalInstructions(), - ...formatOriginalTemplates(), + ...formatOriginalPromptWrapper(), ...formatOriginalTabData(), }; saveAgentDraft(data); @@ -244,7 +246,7 @@ deleteAgentDraft(); setTimeout(() => { refreshInstructions(); - refreshTemplates(); + refreshPromptWrapper(); agentFunctionCmp?.refresh(); agentTabsCmp?.refresh(); }); @@ -286,8 +288,8 @@ />
    - diff --git a/src/routes/page/agent/[agentId]/agent-components/agent-prompt-wrapper.svelte b/src/routes/page/agent/[agentId]/agent-components/agent-prompt-wrapper.svelte new file mode 100644 index 00000000..9de521f9 --- /dev/null +++ b/src/routes/page/agent/[agentId]/agent-components/agent-prompt-wrapper.svelte @@ -0,0 +1,91 @@ + + + + + + {#each tabs as tab, idx (idx) } + handleTabClick(tab.name)} + /> + {/each} + + +
    + +
    +
    + +
    +     +     \ No newline at end of file diff --git a/src/routes/page/agent/[agentId]/agent-components/agent-prompt/agent-link.svelte b/src/routes/page/agent/[agentId]/agent-components/agent-prompt/agent-link.svelte new file mode 100644 index 00000000..96c4b6d5 --- /dev/null +++ b/src/routes/page/agent/[agentId]/agent-components/agent-prompt/agent-link.svelte @@ -0,0 +1,185 @@ + + + + + +
    +
    +
    {'Links'}
    +
    +
    +     + + +
    +
    + {'Contents:'} +
    + + +
    addTemplate()} + > + +
    +
    + + {#if inner_links.length > 0} + + {#each inner_links as template, idx (idx) } + selectTemplate(template.uid)} + onDelete={() => deleteTemplate(template.uid)} + onInput={() => handleAgentChange()} + /> + {/each} + + changePrompt(e)} + placeholder="Enter your content" + /> + {/if} +     +     +     \ No newline at end of file diff --git a/src/routes/page/agent/[agentId]/agent-components/agent-template.svelte b/src/routes/page/agent/[agentId]/agent-components/agent-prompt/agent-template.svelte similarity index 95% rename from src/routes/page/agent/[agentId]/agent-components/agent-template.svelte rename to src/routes/page/agent/[agentId]/agent-components/agent-prompt/agent-template.svelte index c192d707..53463c63 100644 --- a/src/routes/page/agent/[agentId]/agent-components/agent-template.svelte +++ b/src/routes/page/agent/[agentId]/agent-components/agent-prompt/agent-template.svelte @@ -15,12 +15,11 @@ export let handleAgentChange = () => {}; export const fetchOriginalTemplates = () => { - return { - templates: inner_templates?.map(x => ({ + const templates = inner_templates?.map(x => ({ name: x.name, content: x.content - })) || [] - }; + })) || []; + return templates; }; export const fetchTemplates = () => { @@ -36,9 +35,7 @@ } } - return { - templates: prompts - }; + return prompts; } export const refresh = () => init(); @@ -124,7 +121,7 @@
    -
    {'Templates'}
    +
    {'Templates'}
     @@ -180,7 +177,7 @@ value={selected_template.content} rows={15} on:input={(e) => changePrompt(e)} - placeholder="Enter your template" + placeholder="Enter your content" /> {/if} diff --git a/src/routes/page/user/me/+page.svelte b/src/routes/page/user/me/+page.svelte index 89d0e02d..77720fa3 100644 --- a/src/routes/page/user/me/+page.svelte +++ b/src/routes/page/user/me/+page.svelte @@ -74,7 +74,8 @@ on:drop={e => handleFileDrop(e)} > Date: Mon, 28 Apr 2025 16:30:29 -0500 Subject: [PATCH 6/8] remove header --- .../agent-components/agent-prompt/agent-link.svelte | 7 ------- .../agent-components/agent-prompt/agent-template.svelte | 7 ------- 2 files changed, 14 deletions(-) diff --git a/src/routes/page/agent/[agentId]/agent-components/agent-prompt/agent-link.svelte b/src/routes/page/agent/[agentId]/agent-components/agent-prompt/agent-link.svelte index 96c4b6d5..1f7bfc98 100644 --- a/src/routes/page/agent/[agentId]/agent-components/agent-prompt/agent-link.svelte +++ b/src/routes/page/agent/[agentId]/agent-components/agent-prompt/agent-link.svelte @@ -118,13 +118,6 @@ - -
    -
    -
    {'Links'}
    -
    -
    -
    diff --git a/src/routes/page/agent/[agentId]/agent-components/agent-prompt/agent-template.svelte b/src/routes/page/agent/[agentId]/agent-components/agent-prompt/agent-template.svelte index 53463c63..8ea6d5ad 100644 --- a/src/routes/page/agent/[agentId]/agent-components/agent-prompt/agent-template.svelte +++ b/src/routes/page/agent/[agentId]/agent-components/agent-prompt/agent-template.svelte @@ -118,13 +118,6 @@ - -
    -
    -
    {'Templates'}
    -
    -
    -
    From 0c0304e4d9de81d1cdeffcf577084afb961d30d2 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Tue, 29 Apr 2025 14:00:49 -0500 Subject: [PATCH 7/8] refine nav styles --- .../[agentId]/agent-components/agent-prompt-wrapper.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/routes/page/agent/[agentId]/agent-components/agent-prompt-wrapper.svelte b/src/routes/page/agent/[agentId]/agent-components/agent-prompt-wrapper.svelte index 9de521f9..a060dc4f 100644 --- a/src/routes/page/agent/[agentId]/agent-components/agent-prompt-wrapper.svelte +++ b/src/routes/page/agent/[agentId]/agent-components/agent-prompt-wrapper.svelte @@ -70,7 +70,7 @@ {#each tabs as tab, idx (idx) } Date: Sun, 4 May 2025 23:20:34 -0500 Subject: [PATCH 8/8] revert --- src/lib/helpers/types/agentTypes.js | 8 - src/routes/page/agent/[agentId]/+page.svelte | 33 ++-- .../agent-prompt-wrapper.svelte | 91 --------- .../agent-prompt/agent-link.svelte | 178 ------------------ .../{agent-prompt => }/agent-template.svelte | 7 + 5 files changed, 22 insertions(+), 295 deletions(-) delete mode 100644 src/routes/page/agent/[agentId]/agent-components/agent-prompt-wrapper.svelte delete mode 100644 src/routes/page/agent/[agentId]/agent-components/agent-prompt/agent-link.svelte rename src/routes/page/agent/[agentId]/agent-components/{agent-prompt => }/agent-template.svelte (96%) diff --git a/src/lib/helpers/types/agentTypes.js b/src/lib/helpers/types/agentTypes.js index ed9701c5..aa199b54 100644 --- a/src/lib/helpers/types/agentTypes.js +++ b/src/lib/helpers/types/agentTypes.js @@ -10,13 +10,6 @@ * @property {string} content */ -/** - * @typedef {Object} AgentLink - * @property {string?} [uid] - * @property {string} name - * @property {string} content - */ - /** * @typedef {Object} AgentLlmConfig * @property {boolean} is_inherit - Inherited from default Agent settings @@ -72,7 +65,6 @@ * @property {import('$pluginTypes').PluginDefModel} plugin * @property {FunctionDef[]} functions * @property {AgentTemplate[]} templates - * @property {AgentLink[]} links * @property {Object[]} responses * @property {RoutingRule[]} routing_rules * @property {AgentRule[]} rules diff --git a/src/routes/page/agent/[agentId]/+page.svelte b/src/routes/page/agent/[agentId]/+page.svelte index e8a812d1..0ca18306 100644 --- a/src/routes/page/agent/[agentId]/+page.svelte +++ b/src/routes/page/agent/[agentId]/+page.svelte @@ -19,7 +19,7 @@ import { goto } from '$app/navigation'; import { AgentExtensions } from '$lib/helpers/utils/agent'; import LocalStorageManager from '$lib/helpers/utils/storage-manager'; - import AgentPromptWrapper from './agent-components/agent-prompt-wrapper.svelte'; + import AgentTemplate from './agent-components/agent-template.svelte'; /** @type {import('$agentTypes').AgentModel} */ let agent; @@ -28,7 +28,7 @@ /** @type {any} */ let agentInstructionCmp = null; /** @type {any} */ - let agentPromptWrapperCmp = null; + let agentTemplateCmp = null; /** @type {any} */ let agentTabsCmp = null; /** @type {import('$agentTypes').AgentModel} */ @@ -94,7 +94,7 @@ function handleAgentUpdate() { fetchJsonContent(); fetchInstructions(); - fetchPromptWrapper(); + fetchTemplates(); fetchTabData(); agent = { @@ -115,7 +115,7 @@ isComplete = true; deleteAgentDraft(); refreshInstructions(); - refreshPromptWrapper(); + refreshTemplates(); setTimeout(() => { isComplete = false; }, duration); @@ -164,22 +164,19 @@ } // Templates - function formatOriginalPromptWrapper() { - const obj = agentPromptWrapperCmp?.fetchOriginalPromptWrapperData(); + function formatOriginalTemplates() { + const templates = agentTemplateCmp?.fetchOriginalTemplates(); return { - templates: obj.templates || [], - links: obj.links || [] + templates: templates || [] } } - function fetchPromptWrapper() { - const obj = agentPromptWrapperCmp?.fetchPromptWrapperData(); - agent.templates = obj.templates || []; - agent.links = obj.links || []; + function fetchTemplates() { + agent.templates = agentTemplateCmp?.fetchTemplates();; } - function refreshPromptWrapper() { - agentPromptWrapperCmp?.refresh(); + function refreshTemplates() { + agentTemplateCmp?.refresh(); } @@ -234,7 +231,7 @@ ...agent, ...formatJsonContent(), ...formatOriginalInstructions(), - ...formatOriginalPromptWrapper(), + ...formatOriginalTemplates(), ...formatOriginalTabData(), }; saveAgentDraft(data); @@ -246,7 +243,7 @@ deleteAgentDraft(); setTimeout(() => { refreshInstructions(); - refreshPromptWrapper(); + refreshTemplates(); agentFunctionCmp?.refresh(); agentTabsCmp?.refresh(); }); @@ -288,8 +285,8 @@ />
    - diff --git a/src/routes/page/agent/[agentId]/agent-components/agent-prompt-wrapper.svelte b/src/routes/page/agent/[agentId]/agent-components/agent-prompt-wrapper.svelte deleted file mode 100644 index a060dc4f..00000000 --- a/src/routes/page/agent/[agentId]/agent-components/agent-prompt-wrapper.svelte +++ /dev/null @@ -1,91 +0,0 @@ - - - - - - {#each tabs as tab, idx (idx) } - handleTabClick(tab.name)} - /> - {/each} - - -
    - -
    -
    - -
    -     -     \ No newline at end of file diff --git a/src/routes/page/agent/[agentId]/agent-components/agent-prompt/agent-link.svelte b/src/routes/page/agent/[agentId]/agent-components/agent-prompt/agent-link.svelte deleted file mode 100644 index 1f7bfc98..00000000 --- a/src/routes/page/agent/[agentId]/agent-components/agent-prompt/agent-link.svelte +++ /dev/null @@ -1,178 +0,0 @@ - - - - - - -
    -
    - {'Contents:'} -
    - - -
    addTemplate()} - > - -
    -
    - - {#if inner_links.length > 0} - - {#each inner_links as template, idx (idx) } - selectTemplate(template.uid)} - onDelete={() => deleteTemplate(template.uid)} - onInput={() => handleAgentChange()} - /> - {/each} - - changePrompt(e)} - placeholder="Enter your content" - /> - {/if} -     -     -     \ No newline at end of file diff --git a/src/routes/page/agent/[agentId]/agent-components/agent-prompt/agent-template.svelte b/src/routes/page/agent/[agentId]/agent-components/agent-template.svelte similarity index 96% rename from src/routes/page/agent/[agentId]/agent-components/agent-prompt/agent-template.svelte rename to src/routes/page/agent/[agentId]/agent-components/agent-template.svelte index 8ea6d5ad..70255c2d 100644 --- a/src/routes/page/agent/[agentId]/agent-components/agent-prompt/agent-template.svelte +++ b/src/routes/page/agent/[agentId]/agent-components/agent-template.svelte @@ -118,6 +118,13 @@ + +
    +
    +
    {'Templates'}
    +
    +
    +