Features/realtime chat

src/lib/common/ProfileDropdown.svelte

@@ -42,7 +42,8 @@
 	>
 		<img
 			class="rounded-circle header-profile-user"
-			src={`${buildUrl(PUBLIC_SERVICE_URL, user?.avatar)}?access_token=${$userStore?.token}`}
+			src={`${user?.avatar && $userStore?.token ?
+				`${buildUrl(PUBLIC_SERVICE_URL, user?.avatar)}?access_token=${$userStore?.token}` : ''}`}
 			alt=""
 			on:error={e => handleAvatarLoad(e)}
 		/>

src/lib/common/audio-player/AudioPlayer.svelte

@@ -242,7 +242,7 @@
     if (loop === "none") {
       if (order === "list") {
         if ($playList.playingIndex < audios.length - 1) {
-          const promise = buildNextSongPromise(nextIdx);
+          const promise = buildNextAudioPromise(nextIdx);
           promise.then(() => play());
         } else {
           $playList.playingIndex = ($playList.playingIndex + 1) % audios.length;
@@ -257,21 +257,21 @@
         } else {
           targetIdx = randomIdx;
         }
-        const promise = buildNextSongPromise(targetIdx);
+        const promise = buildNextAudioPromise(targetIdx);
         promise.then(() => play());
       }
     } else if (loop === "one") {
       player.currentTime = 0;
     } else if (loop === "all") {
-      const promise = buildNextSongPromise(nextIdx);
+      const promise = buildNextAudioPromise(nextIdx);
       promise.then(() => play());
     }
   };
 
   /**
    * @param {number} idx
    */
-  function buildNextSongPromise(idx) {
+  function buildNextAudioPromise(idx) {
     return new Promise((/** @type {any} */ resolve) => {
       $playList.playingIndex = idx;
       player.currentTime = 0;
@@ -287,8 +287,8 @@
   /**
    * @param {number} idx
    */
-  function switchSong(idx) {
-    const promise = buildNextSongPromise(idx);
+  function switchAudio(idx) {
+    const promise = buildNextAudioPromise(idx);
     if (autoPlayNextOnClick) {
       promise.then(() => {
         play();
@@ -490,7 +490,7 @@
         {#each $audioList as song, idx}
           <!-- svelte-ignore a11y-click-events-have-key-events -->
           <!-- svelte-ignore a11y-no-noninteractive-element-interactions -->
-          <li on:click={() => switchSong(idx) }>
+          <li on:click={() => switchAudio(idx) }>
           {#if idx === $playList.playingIndex}
             <span class="aplayer-list-cur" />
           {/if}

src/lib/helpers/realtime/pcmProcessor.js

@@ -0,0 +1,61 @@
+
+export const AudioRecordingWorklet = `
+class AudioProcessingWorklet extends AudioWorkletProcessor {
+
+  // send and clear buffer every 2048 samples, 
+  // which at 16khz is about 8 times a second,
+  // or at 24khz is about 11 times a second
+  buffer = new Int16Array(2048);
+
+  // current write index
+  bufferWriteIndex = 0;
+
+  speaking = false;
+  threshold = 0.1;
+
+  constructor() {
+    super();
+  }
+
+  /**
+   * @param inputs Float32Array[][] [input#][channel#][sample#] so to access first inputs 1st channel inputs[0][0]
+   * @param outputs Float32Array[][]
+   */
+  process(inputs) {
+    if (inputs[0].length) {
+      const channel0 = inputs[0][0];
+      this.speaking = channel0.some(sample => Math.abs(sample) >= this.threshold);
+      this.processChunk(channel0);
+    }
+    return true;
+  }
+
+  sendAndClearBuffer(){
+    this.port.postMessage({
+      event: "chunk",
+      data: {
+        speaking: this.speaking,
+        int16arrayBuffer: this.buffer.slice(0, this.bufferWriteIndex).buffer,
+      },
+    });
+    this.bufferWriteIndex = 0;
+  }
+
+  processChunk(float32Array) {
+    const l = float32Array.length;
+
+    for (let i = 0; i < l; i++) {
+      // convert float32 -1 to 1 to int16 -32768 to 32767
+      const int16Value = float32Array[i] * 32768;
+      this.buffer[this.bufferWriteIndex++] = int16Value;
+      if (this.bufferWriteIndex >= this.buffer.length) {
+        this.sendAndClearBuffer();
+      }
+    }
+
+    if (this.bufferWriteIndex >= this.buffer.length) {
+      this.sendAndClearBuffer();
+    }
+  }
+}
+`;

src/lib/scss/custom/pages/_agent.scss

@@ -100,7 +100,7 @@
 
             .agent-prompt-header {
                 background-color: white;
-                padding: 20px;
+                padding: 15px;
             }
 
             .agent-prompt-body {

src/lib/services/llm-realtime-service.js

@@ -1,7 +1,6 @@
 import { endpoints } from '$lib/services/api-endpoints.js';
 import { replaceUrl } from '$lib/helpers/http';
 import axios from 'axios';
-import { json } from '@sveltejs/kit';
 
 export const llmRealtime = {
     /** @type {RTCPeerConnection} */

src/lib/services/realtime-chat-service.js

@@ -0,0 +1,237 @@
+import { PUBLIC_SERVICE_URL } from "$env/static/public";
+import { AudioRecordingWorklet } from "$lib/helpers/realtime/pcmProcessor";
+
+// @ts-ignore
+const AudioContext = window.AudioContext || window.webkitAudioContext;
+
+const sampleRate = 24000;
+
+/** @type {AudioContext} */
+let audioCtx = new AudioContext();
+
+/** @type {any[]} */
+let audioQueue = [];
+
+/** @type {boolean} */
+let isPlaying = false;
+
+/** @type {WebSocket | null} */
+let socket = null;
+
+/** @type {MediaStream | null} */
+let mediaStream = null;
+
+/** @type {AudioWorkletNode | null} */
+let workletNode = null;
+
+/** @type {MediaStreamAudioSourceNode | null} */
+let micSource = null;
+
+export const realtimeChat = {
+
+    /**
+     * @param {string} agentId
+     * @param {string} conversationId
+     */
+    start(agentId, conversationId) {
+        reset();
+        const wsUrl = buildWebsocketUrl();
+        socket = new WebSocket(`${wsUrl}/chat/stream/${agentId}/${conversationId}`);
+
+        socket.onopen = async () => {
+            console.log("WebSocket connected");
+
+            socket?.send(JSON.stringify({
+                event: "start"
+            }));
+
+            mediaStream = await navigator.mediaDevices.getUserMedia({ audio: true });
+            audioCtx = new AudioContext({ sampleRate: sampleRate });
+
+            const workletName = "audio-recorder-worklet";
+            const src = createWorkletFromSrc(workletName, AudioRecordingWorklet);
+            await audioCtx.audioWorklet.addModule(src);
+
+            workletNode = new AudioWorkletNode(audioCtx, workletName);
+            micSource = audioCtx.createMediaStreamSource(mediaStream);
+            micSource.connect(workletNode);
+
+            workletNode.port.onmessage = event => {
+                const arrayBuffer = event.data.data.int16arrayBuffer;
+                if (arrayBuffer && socket?.readyState === WebSocket.OPEN) {
+                    if (event.data.data.speaking) {
+                        reset();
+                    }
+                    const arrayBufferString = arrayBufferToBase64(arrayBuffer);
+                    socket.send(JSON.stringify({
+                        event: 'media',
+                        body: {
+                            payload: arrayBufferString
+                        }
+                    }));
+                }
+            };
+        };
+
+        socket.onmessage = (/** @type {MessageEvent} */ e) => {
+            try {
+                const json = JSON.parse(e.data);
+                if (json.event === 'media' && !!json.media.payload) {
+                    const data = json.media.payload;
+                    enqueueAudioChunk(data);
+                }
+            } catch {
+                // console.error('Error when parsing message');
+            }
+        };
+
+        socket.onclose = () => {
+            console.log("Websocket closed");
+        };
+
+        socket.onerror = (/** @type {Event} */ e) => {
+            console.error('WebSocket error', e);
+        };
+    },
+
+    stop() {
+        reset();
+
+        if (mediaStream) {
+            mediaStream.getTracks().forEach(t => t.stop());
+            mediaStream = null;
+        }
+
+        if (workletNode) {
+            micSource?.disconnect(workletNode);
+            workletNode.port.close();
+            workletNode.disconnect();
+            micSource = null;
+            workletNode = null;
+        }
+
+        if (socket?.readyState === WebSocket.OPEN) {
+            socket.send(JSON.stringify({
+                event: 'disconnect'
+            }));
+            socket.close();
+            socket = null;
+        }
+    }
+};
+
+
+function buildWebsocketUrl() {
+    let url = '';
+    const host = PUBLIC_SERVICE_URL.split('://');
+
+    if (PUBLIC_SERVICE_URL.startsWith('https')) {
+        url = `wss:${host[1]}`;
+    } else if (PUBLIC_SERVICE_URL.startsWith('http')) {
+        url = `ws:${host[1]}`;
+    }
+
+    return url;
+}
+
+function reset() {
+    isPlaying = false;
+    audioQueue = [];
+}
+
+/**
+ * @param {string} base64Audio
+ */
+function enqueueAudioChunk(base64Audio) {
+    const arrayBuffer = base64ToArrayBuffer(base64Audio);
+    const float32Data = convert16BitPCMToFloat32(arrayBuffer);
+
+    const audioBuffer = audioCtx.createBuffer(1, float32Data.length, sampleRate);
+    audioBuffer.getChannelData(0).set(float32Data);
+    audioQueue.push(audioBuffer);
+
+    if (!isPlaying) {
+        playNext();
+    }
+}
+
+function playNext() {
+    if (audioQueue.length === 0) {
+      isPlaying = false;
+      return;
+    }
+
+    isPlaying = true;
+    const buffer = audioQueue.shift();
+
+    const source = audioCtx.createBufferSource();
+    source.buffer = buffer;
+    source.connect(audioCtx.destination);
+    source.onended = () => {
+        playNext();
+    };
+    source.start();
+}
+
+
+/**
+ * @param {string} workletName
+ * @param {string} workletSrc
+ */
+function createWorkletFromSrc(workletName, workletSrc) {
+    const script = new Blob(
+      [`registerProcessor("${workletName}", ${workletSrc})`],
+      {
+        type: "application/javascript",
+      },
+    );
+
+    return URL.createObjectURL(script);
+};
+
+
+/**
+ * @param {ArrayBuffer} buffer
+ */
+function arrayBufferToBase64(buffer) {
+    var binary = "";
+    var bytes = new Uint8Array(buffer);
+    var len = bytes.byteLength;
+    for (var i = 0; i < len; i++) {
+      binary += String.fromCharCode(bytes[i]);
+    }
+    return btoa(binary);
+};
+
+/**
+ * @param {string} base64
+ */
+function base64ToArrayBuffer(base64) {
+    const binaryStr = atob(base64);
+    const len = binaryStr.length;
+    const bytes = new Uint8Array(len);
+
+    for (let i = 0; i < len; i++) {
+      bytes[i] = binaryStr.charCodeAt(i);
+    }
+    return bytes.buffer;
+};
+
+/**
+ * @param {ArrayBuffer} buffer
+ */
+function convert16BitPCMToFloat32(buffer) {
+    const chunk = new Uint8Array(buffer);
+    const output = new Float32Array(chunk.length / 2);
+    const dataView = new DataView(chunk.buffer);
+
+    for (let i = 0; i< chunk.length / 2; i++) {
+        try {
+            const int16 = dataView.getInt16(i * 2, true);
+            output[i] = int16 / 32768;
+        } catch (e) {
+            console.error(e);
+        }
+    }
+    return output;
+};