import {AudioFormatReader} from './3las.formatreader.js' import LiveAudioPlayer from './3las.liveaudioplayer' import WebSocketClient from './3las.websocketclient' var _3LAS_Settings = /** @class */ (function () { function _3LAS_Settings() { this.SocketHost = document.location.hostname ? document.location.hostname : "127.0.0.1"; this.SocketPort = 0; this.Formats = new Array(); this.MaxVolume = 1.0; this.AutoCorrectSpeed = false; this.InitialBufferLength = 1.0 / 3.0; this.isAndroid = undefined; this.isIOS = undefined; this.isIPadOS= undefined; this.isWindows= undefined; this.isLinux= undefined; this.isBSD= undefined; this.isMacOSX= undefined; this.isInternetExplorer= undefined; this.isEdge= undefined; this.isSafari= undefined; this.isOpera= undefined; this.isChrome= undefined; this.isFirefox= undefined; this.webkitVer= undefined; this.isNativeChrome= undefined; this.BrowserName= undefined; this.OSName= undefined; var ua = navigator.userAgent.toLowerCase(); this.isAndroid = (ua.match('android') ? true : false); this.isIOS = (ua.match(/(iphone|ipod)/g) ? true : false); this.isIPadOS = ((ua.match('ipad') || (navigator.platform == 'MacIntel' && navigator.maxTouchPoints > 1)) ? true : false); this.isWindows = (ua.match('windows') ? true : false); this.isLinux = (ua.match('android') ? false : (ua.match('linux') ? true : false)); this.isBSD = (ua.match('bsd') ? true : false); this.isMacOSX = !this.isIOS && !this.isIPadOS && (ua.match('mac osx') ? true : false); this.isInternetExplorer = (ua.match('msie') ? true : false); this.isEdge = (ua.match('edg') ? true : false); this.isSafari = (ua.match(/(chromium|chrome|crios)/g) ? false : (ua.match('safari') ? true : false)); this.isOpera = (ua.match('opera') ? true : false); this.isChrome = !this.isSafari && (ua.match(/(chromium|chrome|crios)/g) ? true : false); this.isFirefox = (ua.match('like gecko') ? false : (ua.match(/(gecko|fennec|firefox)/g) ? true : false)); this.webkitVer = parseInt((/WebKit\/([0-9]+)/.exec(navigator.appVersion) || ["", "0"])[1], 10) || void 0; // also match AppleWebKit this.isNativeChrome = this.isAndroid && this.webkitVer <= 537 && navigator.vendor.toLowerCase().indexOf('google') == 0; this.BrowserName = "Unknown"; if (this.isInternetExplorer) this.BrowserName = "IE"; else if (this.isEdge) this.BrowserName = "Edge"; else if (this.isSafari) this.BrowserName = "Safari"; else if (this.isOpera) this.BrowserName = "Opera"; else if (this.isChrome) this.BrowserName = "Chrome"; else if (this.isFirefox) this.BrowserName = "Firefox"; else if (this.isNativeChrome) this.BrowserName = "NativeChrome"; else this.BrowserName = "Unknown"; this.OSName = "Unknown"; if (this.isAndroid) this.OSName = "Android"; else if (this.isIOS) this.OSName = "iOS"; else if (this.isIPadOS) this.OSName = "iPadOS"; else if (this.isWindows) this.OSName = "Windows"; else if (this.isLinux) this.OSName = "Linux"; else if (this.isBSD) this.OSName = "BSD"; else if (this.isMacOSX) this.OSName = "MacOSX"; else this.OSName = "Unknown"; } return _3LAS_Settings; }()); var _3LAS = /** @class */ (function () { function _3LAS(logger, settings, VUE, SID) { this.TODO_CNT = 0; // 꼭 지워야 됌 this.pingTimeout = 0; this.SID = SID; this.STATE = { WAVS_STREAMING_LINK_OPEN : 'a', WAVS_SLINK_VOC_STARTING : 'b', WAVS_SLINK_VOC_CONTINUE :'c', WAVS_SLINK_VOC_END : 'd', WAVS_SLINK_RECOG_STARTING :'e', WAVS_SLINK_RECOG_CONTINUE :'f', WAVS_SLINK_RECOG_END :'g'}; this.audioInput = null; this.audioNode = null; this.VUE_STORE = VUE; this.recordSampleRate = 8000; this.recordBufferSize = 2048; this.Logger = logger; if (!this.Logger) { this.Logger = new Logging(null, null); } this.Settings = settings; // Create audio context if (typeof AudioContext !== "undefined") this.Audio = new AudioContext(); else if (typeof webkitAudioContext !== "undefined") this.Audio = new webkitAudioContext(); else if (typeof mozAudioContext !== "undefined") this.Audio = new mozAudioContext(); else { this.Logger.Log('3LAS: Browser does not support "AudioContext".'); throw new Error(); } this.Logger.Log("Detected: " + (settings.OSName == "MacOSX" ? "Mac OSX" : (settings.OSName == "Unknown" ? "Unknown OS" : settings.OSName)) + ", " + (settings.BrowserName == "IE" ? "Internet Explorer" : (settings.BrowserName == "NativeChrome" ? "Chrome legacy" : (settings.BrowserName == "Unknown" ? "Unknown Browser" : settings.BrowserName)))); try { this.Player = new LiveAudioPlayer(this.Audio, this.Logger, this.Settings.MaxVolume, this.Settings.InitialBufferLength, this.Settings.AutoCorrectSpeed); this.Logger.Log("Init of LiveAudioPlayer succeeded"); } catch (e) { this.Logger.Log("Init of LiveAudioPlayer failed: " + e); throw new Error(); } this.SelectedMime = ""; this.SelectedPort = '9601'; for (var i = 0; i < this.Settings.Formats.length; i++) { if (!AudioFormatReader.CanDecodeTypes([this.Settings.Formats[i].Mime])) continue; this.SelectedMime = this.Settings.Formats[i].Mime; this.SelectedPort = this.Settings.Formats[i].Port; break; } if (this.SelectedMime == "" || this.SelectedPort == 0) { this.Logger.Log("None of the available MIME types are supported."); throw new Error(); } this.Logger.Log("Using MIME: " + this.SelectedMime + " on port: " + this.SelectedPort.toString()); this.PacketModCounter = 0; this.LastCheckTime = 0; this.FocusChecker = 0; this.checkServerState = this.webSocketHeartBeat.bind(this); } _3LAS.prototype.Test = function(mimeType) { try { this.changeAudioType(mimeType); const url = `wss://${this.Settings.SocketHost}:${this.Settings.SocketPort.toString()}`; this.WebSocket = new WebSocketClient(this.Logger, url, function(){}, function(){}, this.OnReceiveDataByTest.bind(this), function(){}); } catch (e) { this.Logger.Log(`Init of WebSocketClient failed: ${e}`); } } /** * 웨이브(WAVS)서버에서 데이터 받아옴 * 리팩토링 필수 * TODO 꼭 잘라서 사용할것 그냥 사용하면 안됨 */ _3LAS.prototype.OnReceiveDataByTest = function(data) { this.OnSocketDataReady(data); } _3LAS.prototype.Init = function() { try { this.Logger.Log(`WAVS Init ${this.Settings.SocketHost}`); this.WebSocket = new WebSocketClient(this.Logger, this.Settings.SocketHost, this.OnSocketError.bind(this), this.OnSocketConnect.bind(this), this.OnReceiveDataByServer.bind(this), this.OnSocketDisconnect.bind(this)); } catch (e) { this.Logger.Log(`Init of WebSocketClient failed: ${e}`); } } _3LAS.prototype.changeAudioType = function(mimeType) { this.SelectedMime = mimeType; try { this.FormatReader = null; this.FormatReader = AudioFormatReader.Create(this.SelectedMime, this.Audio, this.Logger, this.OnReaderError.bind(this), this.Player.CheckBeforeDecode, this.OnReaderDataReady.bind(this), AudioFormatReader.DefaultSettings()); this.PacketModCounter = 0; this.LastCheckTime = 0; this.FocusChecker = 0; } catch (e) { this.Logger.Log(`Init of AudioFormatReader failed: ${e}`); } } Object.defineProperty(_3LAS.prototype, "Volume", { get: function () { return this.Player.Volume / this.Settings.MaxVolume; }, set: function (value) { this.Player.Volume = value * this.Settings.MaxVolume; }, enumerable: true, configurable: true }); // Callback functions from format reader _3LAS.prototype.OnReaderError = function () { this.Logger.Log("Reader error: Decoding failed."); }; _3LAS.prototype.OnReaderDataReady = function () { while (this.FormatReader.SamplesAvailable()) { this.Player.PushBuffer(this.FormatReader.PopSamples()); } }; // Callback function from socket connection _3LAS.prototype.OnSocketError = function (message) { clearTimeout(this.pingTimeout); let msg= `WAVS server connection fail. message[${message}]`; this.Logger.Log(msg); const errorResult = {code : 'socketError', msg : msg}; this.VUE_STORE.commit('STREAM_SERVER_ERROR', errorResult); }; _3LAS.prototype.webSocketHeartBeat = function () { clearTimeout(this.pingTimeout); this.pingTimeout = setTimeout(() => { if(this.WebSocket.CheckServer()){ this.OnSocketDisconnect(); return; } this.checkServerState(); },1000); }; _3LAS.prototype.OnSocketConnect = function () { //this.StartFocusChecker(); //this.checkServerState(); this.Logger.Log(`OnSocketConnect code:[${this.STATE.WAVS_STREAMING_LINK_OPEN}], SID:[${this.SID}]`); this.WebSocket.Send({code : this.STATE.WAVS_STREAMING_LINK_OPEN, SID:this.SID}, {code:1, SID:20}); this.VUE_STORE.state.IsConnected = true; //this.ReadyToStreaming(); }; // TODO 커넥션 끊겼을때 어찌 할지 _3LAS.prototype.OnSocketDisconnect = function () { let msg= `WAVS server disconnected. `; try{ this.StopFocusChecker(); this.FormatReader.Reset(); this.Player.Reset(); clearTimeout(this.pingTimeout); }catch(e) { this.Logger.Log(e); } this.Logger.Log(msg); const errorResult = {code : 'socketError', msg : msg}; this.VUE_STORE.commit('STREAM_SERVER_ERROR', errorResult); }; /** * 웨이브(WAVS)서버에서 데이터 받아옴 * 리팩토링 필수 * TODO 꼭 잘라서 사용할것 그냥 사용하면 안됨 */ _3LAS.prototype.OnReceiveDataByServer = function(data) { let bufView = new Uint8Array(data); let header = String.fromCharCode.apply(null, bufView.slice(0,1)); let body = bufView.slice(1, bufView.length); //this.Logger.Log(`WAVS DATA RECEIVED ::OnReceiveDataByServer:: header[${header}], body[${body}]`); switch (header) { case this.STATE.WAVS_STREAMING_LINK_OPEN: this.OnStreamingLinkOpen(body.buffer); break; case this.STATE.WAVS_SLINK_VOC_STARTING: this.ReadyToStreaming(body.buffer); break; case this.STATE.WAVS_SLINK_VOC_CONTINUE: this.OnSocketDataReady(body.buffer); break; case this.STATE.WAVS_SLINK_VOC_END: this.OnStreamingEnd(body.buffer); break; } } _3LAS.prototype.OnStreamingLinkOpen = function (data) { let bufView = new Uint8Array(data); if(bufView[0] !== 0){ const message = `OnStreamingLinkOpen fail!! ${bufView[0]}`; const result = {code : 'failValue', msg:message}; this.VUE_STORE.commit('STREAM_ERROR_CALLBACK', result); } }; _3LAS.prototype.ReadyToStreaming = function (data) { //TODO MIME TYPE 추출 /* let bufView = new Uint8Array(data); let mime_type = String.fromCharCode.apply(null, bufView.slice(0,3)); this.Logger.Log(`ReadyToStreaming${mime_type}`);*/ this.Logger.Log(`ReadyToStreaming`) this.changeAudioType('audio/wav'); this.StartFocusChecker(); }; _3LAS.prototype.StartFocusChecker = function () { if (!this.FocusChecker) { this.LastCheckTime = Date.now(); this.FocusChecker = window.setInterval(this.CheckFocus.bind(this), 2000); } }; _3LAS.prototype.StopFocusChecker = function () { if (this.FocusChecker) { window.clearInterval(this.FocusChecker); this.FocusChecker = 0; } }; _3LAS.prototype.CheckFocus = function () { var checkTime = Date.now(); // Check if focus was lost if (checkTime - this.LastCheckTime > 10000) { // If so, drop all samples in the buffer this.Logger.Log("Focus lost, purging format reader."); this.FormatReader.PurgeData(); } this.LastCheckTime = checkTime; }; _3LAS.prototype.OnSocketDataReady = function (data) { /* if(this.TODO_CNT === 0){ this.TODO_CNT ++; return; } */ this.PacketModCounter++; if (this.PacketModCounter > 100) { this.PacketModCounter = 0; } this.FormatReader.PushData(new Uint8Array(data)); }; _3LAS.prototype.MobileUnmute = function () { var amplification = this.Audio.createGain(); // Set volume to max amplification.gain.value = 1.0; // Connect gain node to context amplification.connect(this.Audio.destination); // Create one second buffer with silence var audioBuffer = this.Audio.createBuffer(2, this.Audio.sampleRate, this.Audio.sampleRate); // Create new audio source for the buffer var sourceNode = this.Audio.createBufferSource(); // Make sure the node deletes itself after playback sourceNode.onended = function (_ev) { sourceNode.disconnect(); amplification.disconnect(); }; // Pass audio data to source sourceNode.buffer = audioBuffer; // Connect the source to the gain node sourceNode.connect(amplification); // Play source sourceNode.start(); }; // 사용자에게 정상적으로 스트리밍이 끝났다고 알려줘야 함 _3LAS.prototype.OnStreamingEnd = function (data) { this.Logger.Log("_3LAS.OnStreamingEnd"); this.StopFocusChecker(); this.FormatReader.Reset(); this.Player.Reset(); this.TODO_CNT =0; }; _3LAS.prototype.VoiceStart = function(){ this.Init(); this.WebSocket.Send({code:this.STATE.WAVS_SLINK_VOC_STARTING},{code:1}); } _3LAS.prototype.RecordStart = function(currentViewId) { this.Logger.Log(`RecordStart ${currentViewId}`); this.Logger.Log(`code:[${this.STATE.WAVS_SLINK_RECOG_STARTING}], viewPath:[${currentViewId}]`); this.WebSocket.Send({code:this.STATE.WAVS_SLINK_RECOG_STARTING, path:currentViewId}, {code:1, path:currentViewId.length}); navigator.mediaDevices.getUserMedia({audio: true}).then(this._onMicrophoneCaptured.bind(this)).catch(this._onMicrophoneError.bind(this)); } _3LAS.prototype._onMicrophoneCaptured = function(microphone) { window.localStream = microphone; this.audioInput = this.Audio.createMediaStreamSource(microphone); if(this.Audio.sampleRate === 48000 || this.Audio.sampleRate === 44100 ) this.recordBufferSize = 16384; if (this.Audio.createJavaScriptNode){ this.audioNode = this.Audio.createJavaScriptNode(this.recordBufferSize, 1, 1); }else if (this.Audio.createScriptProcessor){ this.audioNode = this.Audio.createScriptProcessor(this.recordBufferSize, 1, 1); }else{ throw 'WebAudio not supported!'; } this.audioNode.connect(this.Audio.destination); this.audioInput.connect(this.audioNode); this.audioNode.onaudioprocess = this._onAudioProcess.bind(this); this.recordSampleRate = this.Audio.sampleRate; } _3LAS.prototype._onMicrophoneError = function(e) { this.Logger.Log(e); } _3LAS.prototype._onAudioProcess = function(e) { var interleaved; if ( this.recordSampleRate == 8000 ){ interleaved = new Float32Array(e.inputBuffer.getChannelData(0)); } else{ interleaved = this._compressTo8000( new Float32Array(e.inputBuffer.getChannelData(0)) ); } let buffer = new ArrayBuffer(interleaved.length * 2); let view = new DataView(buffer); // write the PCM samples let index = 0; let volume = 1; for ( let i = 0; i < interleaved.length; i++ ) { view.setInt16(index, interleaved[i] * (0x7FFF * volume), true); index += 2; } let sendBuffer = new ArrayBuffer(1 + buffer.byteLength); let sendData = new Uint8Array(sendBuffer); sendData[0] = this.STATE.WAVS_SLINK_RECOG_CONTINUE.charCodeAt(0); let body = new Uint8Array(buffer); sendData.set(body ,1); this.Logger.Log(sendData); this.WebSocket.SendRawData(sendData); } _3LAS.prototype.RecordStop = function() { this.RecordStreamStop(); this.audioInput.disconnect(); this.audioNode.disconnect(); this.WebSocket.Send({code : this.STATE.WAVS_SLINK_RECOG_END}, {code:1}); } _3LAS.prototype.RecordStreamStop = function() { if (window.localStream) { window.localStream.getAudioTracks().forEach(function(track) { track.stop(); }); window.localStream.getVideoTracks().forEach(function(track) { track.stop(); }); window.localStream = null; } } _3LAS.prototype.ChatbotStart = function(param) { console.log(JSON.stringify(param)); } _3LAS.prototype._compressTo8000 = function(inputL){ var compression = this.recordSampleRate / 8000; var length = inputL.length / compression; var result = new Float32Array(length); var index = 0, inputIndex = 0; while (index < length) { result[index++] = inputL[Math.round(inputIndex)]; inputIndex += compression; } return result; } return _3LAS; }()); export { _3LAS, _3LAS_Settings }