return timestamps for WebAssembly (#737)

k2-fsa · Apr 5, 2024 · c1c0f5b · c1c0f5b
1 parent dbff2ea
commit c1c0f5b
Show file tree

Hide file tree

Showing 20 changed files with 88 additions and 34 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -1,7 +1,7 @@
 cmake_minimum_required(VERSION 3.13 FATAL_ERROR)
 project(sherpa-onnx)
 
-set(SHERPA_ONNX_VERSION "1.9.15")
+set(SHERPA_ONNX_VERSION "1.9.16")
 
 # Disable warning about
 #

diff --git a/build-aarch64-linux-gnu.sh b/build-aarch64-linux-gnu.sh
@@ -57,7 +57,7 @@ cmake \
   -DSHERPA_ONNX_ENABLE_CHECK=OFF \
   -DSHERPA_ONNX_ENABLE_PORTAUDIO=OFF \
   -DSHERPA_ONNX_ENABLE_JNI=OFF \
-  -DSHERPA_ONNX_ENABLE_C_API=OFF \
+  -DSHERPA_ONNX_ENABLE_C_API=ON \
   -DSHERPA_ONNX_ENABLE_WEBSOCKET=OFF \
   -DCMAKE_TOOLCHAIN_FILE=../toolchains/aarch64-linux-gnu.toolchain.cmake \
   ..

diff --git a/build-arm-linux-gnueabihf.sh b/build-arm-linux-gnueabihf.sh
@@ -52,7 +52,7 @@ cmake \
   -DSHERPA_ONNX_ENABLE_CHECK=OFF \
   -DSHERPA_ONNX_ENABLE_PORTAUDIO=OFF \
   -DSHERPA_ONNX_ENABLE_JNI=OFF \
-  -DSHERPA_ONNX_ENABLE_C_API=OFF \
+  -DSHERPA_ONNX_ENABLE_C_API=ON \
   -DSHERPA_ONNX_ENABLE_WEBSOCKET=OFF \
   -DCMAKE_TOOLCHAIN_FILE=../toolchains/arm-linux-gnueabihf.toolchain.cmake \
   ..

diff --git a/build-wasm-simd-nodejs.sh b/build-wasm-simd-nodejs.sh
@@ -57,7 +57,7 @@ cmake \
   -DSHERPA_ONNX_ENABLE_BINARY=OFF \
   -DSHERPA_ONNX_LINK_LIBSTDCPP_STATICALLY=OFF \
   ..
-make -j10
+make -j3
 make install
 
 ls -lh install/bin/wasm/nodejs
diff --git a/nodejs-examples/test-offline-nemo-ctc.js b/nodejs-examples/test-offline-nemo-ctc.js
@@ -100,7 +100,7 @@ fs.createReadStream(waveFilename, {highWaterMark: 4096})
 
       stream.acceptWaveform(recognizer.config.featConfig.sampleRate, flattened);
       recognizer.decode(stream);
-      const text = recognizer.getResult(stream);
+      const text = recognizer.getResult(stream).text;
       console.log(text);
 
       stream.free();

diff --git a/nodejs-examples/test-offline-paraformer.js b/nodejs-examples/test-offline-paraformer.js
@@ -100,7 +100,7 @@ fs.createReadStream(waveFilename, {'highWaterMark': 4096})
 
       stream.acceptWaveform(recognizer.config.featConfig.sampleRate, flattened);
       recognizer.decode(stream);
-      const text = recognizer.getResult(stream);
+      const text = recognizer.getResult(stream).text;
       console.log(text);
 
       stream.free();

diff --git a/nodejs-examples/test-offline-transducer.js b/nodejs-examples/test-offline-transducer.js
@@ -101,7 +101,7 @@ fs.createReadStream(waveFilename, {'highWaterMark': 4096})
 
       stream.acceptWaveform(recognizer.config.featConfig.sampleRate, flattened);
       recognizer.decode(stream);
-      const text = recognizer.getResult(stream);
+      const text = recognizer.getResult(stream).text;
       console.log(text);
 
       stream.free();

diff --git a/nodejs-examples/test-offline-whisper.js b/nodejs-examples/test-offline-whisper.js
@@ -100,7 +100,7 @@ fs.createReadStream(waveFilename, {'highWaterMark': 4096})
 
       stream.acceptWaveform(recognizer.config.featConfig.sampleRate, flattened);
       recognizer.decode(stream);
-      const text = recognizer.getResult(stream);
+      const text = recognizer.getResult(stream).text;
       console.log(text);
 
       stream.free();

diff --git a/nodejs-examples/test-online-paraformer-microphone.js b/nodejs-examples/test-online-paraformer-microphone.js
@@ -86,7 +86,7 @@ ai.on('data', data => {
   }
 
   const isEndpoint = recognizer.isEndpoint(stream);
-  const text = recognizer.getResult(stream);
+  const text = recognizer.getResult(stream).text;
 
   if (text.length > 0 && lastText != text) {
     lastText = text;

diff --git a/nodejs-examples/test-online-paraformer.js b/nodejs-examples/test-online-paraformer.js
@@ -75,7 +75,7 @@ function decode(samples) {
   while (recognizer.isReady(stream)) {
     recognizer.decode(stream);
   }
-  const text = recognizer.getResult(stream);
+  const text = recognizer.getResult(stream).text;
   console.log(text);
 }
 

diff --git a/nodejs-examples/test-online-transducer-microphone.js b/nodejs-examples/test-online-transducer-microphone.js
@@ -88,7 +88,7 @@ ai.on('data', data => {
   }
 
   const isEndpoint = recognizer.isEndpoint(stream);
-  const text = recognizer.getResult(stream);
+  const text = recognizer.getResult(stream).text;
 
   if (text.length > 0 && lastText != text) {
     lastText = text;

diff --git a/nodejs-examples/test-online-transducer.js b/nodejs-examples/test-online-transducer.js
@@ -77,7 +77,7 @@ function decode(samples) {
   while (recognizer.isReady(stream)) {
     recognizer.decode(stream);
   }
-  const text = recognizer.getResult(stream);
+  const text = recognizer.getResult(stream).text;
   console.log(text);
 }
 

diff --git a/nodejs-examples/test-online-zipformer2-ctc-hlg.js b/nodejs-examples/test-online-zipformer2-ctc-hlg.js
@@ -74,7 +74,7 @@ function decode(samples) {
   while (recognizer.isReady(stream)) {
     recognizer.decode(stream);
   }
-  const text = recognizer.getResult(stream);
+  const text = recognizer.getResult(stream).text;
   console.log(text);
 }
 

diff --git a/nodejs-examples/test-online-zipformer2-ctc.js b/nodejs-examples/test-online-zipformer2-ctc.js
@@ -75,7 +75,7 @@ function decode(samples) {
   while (recognizer.isReady(stream)) {
     recognizer.decode(stream);
   }
-  const text = recognizer.getResult(stream);
+  const text = recognizer.getResult(stream).text;
   console.log(text);
 }
 

diff --git a/sherpa-onnx/c-api/c-api.cc b/sherpa-onnx/c-api/c-api.cc
@@ -243,6 +243,20 @@ void DestroyOnlineRecognizerResult(const SherpaOnnxOnlineRecognizerResult *r) {
   }
 }
 
+const char *GetOnlineStreamResultAsJson(
+    const SherpaOnnxOnlineRecognizer *recognizer,
+    const SherpaOnnxOnlineStream *stream) {
+  sherpa_onnx::OnlineRecognizerResult result =
+      recognizer->impl->GetResult(stream->impl.get());
+  std::string json = result.AsJsonString();
+  char *pJson = new char[json.size() + 1];
+  std::copy(json.begin(), json.end(), pJson);
+  pJson[json.size()] = 0;
+  return pJson;
+}
+
+void DestroyOnlineStreamResultJson(const char *s) { delete[] s; }
+
 void Reset(const SherpaOnnxOnlineRecognizer *recognizer,
            const SherpaOnnxOnlineStream *stream) {
   recognizer->impl->Reset(stream->impl.get());
@@ -409,7 +423,7 @@ void DecodeMultipleOfflineStreams(SherpaOnnxOfflineRecognizer *recognizer,
 }
 
 const SherpaOnnxOfflineRecognizerResult *GetOfflineStreamResult(
-    SherpaOnnxOfflineStream *stream) {
+    const SherpaOnnxOfflineStream *stream) {
   const sherpa_onnx::OfflineRecognitionResult &result =
       stream->impl->GetResult();
   const auto &text = result.text;
@@ -444,6 +458,19 @@ void DestroyOfflineRecognizerResult(
   }
 }
 
+const char *GetOfflineStreamResultAsJson(
+    const SherpaOnnxOfflineStream *stream) {
+  const sherpa_onnx::OfflineRecognitionResult &result =
+      stream->impl->GetResult();
+  std::string json = result.AsJsonString();
+  char *pJson = new char[json.size() + 1];
+  std::copy(json.begin(), json.end(), pJson);
+  pJson[json.size()] = 0;
+  return pJson;
+}
+
+void DestroyOfflineStreamResultJson(const char *s) { delete[] s; }
+
 // ============================================================
 // For Keyword Spot
 // ============================================================

diff --git a/sherpa-onnx/c-api/c-api.h b/sherpa-onnx/c-api/c-api.h
@@ -286,6 +286,16 @@ SHERPA_ONNX_API const SherpaOnnxOnlineRecognizerResult *GetOnlineStreamResult(
 SHERPA_ONNX_API void DestroyOnlineRecognizerResult(
     const SherpaOnnxOnlineRecognizerResult *r);
 
+/// Return the result as a json string.
+/// The user has to invoke
+/// DestroyOnlineStreamResultJson()
+/// to free the returned pointer to avoid memory leak
+SHERPA_ONNX_API const char *GetOnlineStreamResultAsJson(
+    const SherpaOnnxOnlineRecognizer *recognizer,
+    const SherpaOnnxOnlineStream *stream);
+
+SHERPA_ONNX_API void DestroyOnlineStreamResultJson(const char *s);
+
 /// Reset an OnlineStream , which clears the neural network model state
 /// and the state for decoding.
 ///
@@ -482,14 +492,22 @@ SHERPA_ONNX_API typedef struct SherpaOnnxOfflineRecognizerResult {
 ///         DestroyOnlineRecognizerResult() to free the returned pointer to
 ///         avoid memory leak.
 SHERPA_ONNX_API const SherpaOnnxOfflineRecognizerResult *GetOfflineStreamResult(
-    SherpaOnnxOfflineStream *stream);
+    const SherpaOnnxOfflineStream *stream);
 
 /// Destroy the pointer returned by GetOfflineStreamResult().
 ///
 /// @param r A pointer returned by GetOfflineStreamResult()
 SHERPA_ONNX_API void DestroyOfflineRecognizerResult(
     const SherpaOnnxOfflineRecognizerResult *r);
 
+/// Return the result as a json string.
+/// The user has to use DestroyOfflineStreamResultJson()
+/// to free the returned pointer to avoid memory leak
+SHERPA_ONNX_API const char *GetOfflineStreamResultAsJson(
+    const SherpaOnnxOfflineStream *stream);
+
+SHERPA_ONNX_API void DestroyOfflineStreamResultJson(const char *s);
+
 // ============================================================
 // For Keyword Spot
 // ============================================================

diff --git a/wasm/asr/CMakeLists.txt b/wasm/asr/CMakeLists.txt
@@ -13,10 +13,14 @@ set(exported_functions
   CreateOnlineRecognizer
   CreateOnlineStream
   DecodeOnlineStream
+  DestroyOfflineStreamResultJson
   DestroyOnlineRecognizer
   DestroyOnlineRecognizerResult
   DestroyOnlineStream
+  DestroyOnlineStreamResultJson
+  GetOfflineStreamResultAsJson
   GetOnlineStreamResult
+  GetOnlineStreamResultAsJson
   InputFinished
   IsEndpoint
   IsOnlineStreamReady

diff --git a/wasm/asr/app-asr.js b/wasm/asr/app-asr.js
@@ -108,7 +108,7 @@ if (navigator.mediaDevices.getUserMedia) {
       }
 
       let isEndpoint = recognizer.isEndpoint(recognizer_stream);
-      let result = recognizer.getResult(recognizer_stream);
+      let result = recognizer.getResult(recognizer_stream).text;
 
 
       if (result.length > 0 && lastResult != result) {

diff --git a/wasm/asr/sherpa-onnx-asr.js b/wasm/asr/sherpa-onnx-asr.js
@@ -661,13 +661,12 @@ class OfflineRecognizer {
   }
 
   getResult(stream) {
-    const r = this.Module._GetOfflineStreamResult(stream.handle);
+    const r = this.Module._GetOfflineStreamResultAsJson(stream.handle);
+    const jsonStr = this.Module.UTF8ToString(r);
+    const ans = JSON.parse(jsonStr);
+    this.Module._DestroyOfflineStreamResultJson(r);
 
-    const textPtr = this.Module.getValue(r, 'i8*');
-    const text = this.Module.UTF8ToString(textPtr);
-
-    this.Module._DestroyOfflineRecognizerResult(r);
-    return text;
+    return ans;
   }
 };
 
@@ -750,11 +749,13 @@ class OnlineRecognizer {
   }
 
   getResult(stream) {
-    const r = this.Module._GetOnlineStreamResult(this.handle, stream.handle);
-    const textPtr = this.Module.getValue(r, 'i8*');
-    const text = this.Module.UTF8ToString(textPtr);
-    this.Module._DestroyOnlineRecognizerResult(r);
-    return text;
+    const r =
+        this.Module._GetOnlineStreamResultAsJson(this.handle, stream.handle);
+    const jsonStr = this.Module.UTF8ToString(r);
+    const ans = JSON.parse(jsonStr);
+    this.Module._DestroyOnlineStreamResultJson(r);
+
+    return ans;
   }
 }
 

diff --git a/wasm/nodejs/CMakeLists.txt b/wasm/nodejs/CMakeLists.txt
@@ -21,22 +21,26 @@ set(exported_functions
   DestroyOnlineRecognizer
   DestroyOnlineRecognizerResult
   DestroyOnlineStream
+  DestroyOnlineStreamResultJson
   GetOnlineStreamResult
+  GetOnlineStreamResultAsJson
   InputFinished
   IsEndpoint
   IsOnlineStreamReady
   Reset
   # non-streaming ASR
-  PrintOfflineRecognizerConfig
+  AcceptWaveformOffline
   CreateOfflineRecognizer
-  DestroyOfflineRecognizer
   CreateOfflineStream
-  DestroyOfflineStream
-  AcceptWaveformOffline
-  DecodeOfflineStream
   DecodeMultipleOfflineStreams
-  GetOfflineStreamResult
+  DecodeOfflineStream
+  DestroyOfflineRecognizer
   DestroyOfflineRecognizerResult
+  DestroyOfflineStream
+  DestroyOfflineStreamResultJson
+  GetOfflineStreamResult
+  GetOfflineStreamResultAsJson
+  PrintOfflineRecognizerConfig
   # online kws
   CreateKeywordSpotter
   DestroyKeywordSpotter