Transcription force alignment & more (#416)

* add wavesurfer-provider * brand new layout for player * refactor pitch contour * clean up * update styl * refactor * update layout * use new layout for video * refactor * may select word * may edit word timestamp * may toggle multiselect words * clean code * improve word region update * improve layout * update layout * add echogarden * fix test * use aligned transcription * fix ipa * some refactor * improve code * implement ipa & translate & lookup * recording play & share * fix * fix post audio * improve layout * may delete recording * may record * fix video player layout * fix player in conversation * render recording along with orignal audio * may custom create region in recording * fix float issue when seekTo * fix recording player * fix load more recordings * fix seekTo * clean up * refactor pitch contour * fix some warnings * upgrade deps * fix group transcription sentence * zoom to fit when segment update * add more hotkeys * update player layout * improve style * play recording overlap audio when comparing * update echogarden dep * add recorded mark on transcription * fix recording pitch contour rendering * improve recording * adjust pitch finder params
ZuodaoTech · Mar 16, 2024 · 90f38e9 · 90f38e9
1 parent fe43755
commit 90f38e9
Show file tree

Hide file tree

Showing 67 changed files with 6,907 additions and 2,652 deletions.
diff --git a/1000-hours/package.json b/1000-hours/package.json
@@ -7,9 +7,9 @@
     "markdown-it-mathjax3": "^4.3.2",
     "markdown-it-sub": "^2.0.0",
     "markdown-it-sup": "^2.0.0",
-    "mermaid": "^10.8.0",
-    "sass": "^1.71.1",
-    "vitepress": "^1.0.0-rc.42",
+    "mermaid": "^10.9.0",
+    "sass": "^1.72.0",
+    "vitepress": "^1.0.0-rc.45",
     "vitepress-plugin-mermaid": "^2.0.16",
     "vue": "^3.4.21"
   },

diff --git a/enjoy/e2e/main.spec.ts b/enjoy/e2e/main.spec.ts
@@ -78,6 +78,16 @@ test("valid ffmpeg command", async () => {
   expect(res).toBeTruthy();
 });
 
+test("validate echogarden align command", async () => {
+  const res = await page.evaluate(() => {
+    return window.__ENJOY_APP__.echogarden.check();
+  });
+  expect(res).toBeTruthy();
+
+  const settings = fs.readJsonSync(path.join(resultDir, "settings.json"));
+  expect(settings.whisper.service).toBe("local");
+});
+
 test("should setup default library path", async () => {
   const settings = fs.readJsonSync(path.join(resultDir, "settings.json"));
   expect(settings.library).not.toBeNull();

diff --git a/enjoy/e2e/renderer.spec.ts b/enjoy/e2e/renderer.spec.ts
@@ -122,9 +122,39 @@ test.describe("with login", async () => {
           },
         });
       });
+    });
 
+    /*
+     * steps:
+     * 1. create a tts conversation
+     * 2. submit a message to the conversation
+     * 3. the speech should auto create
+     */
+    test("tts conversation", async () => {
       // navigate to the conversations page
       await page.getByTestId("sidebar-conversations").click();
+
+      // trigger new conversation modal
+      await page.getByTestId("conversation-new-button").click();
+
+      // create a tts conversation
+      await page.click("[data-testid=conversation-preset-tts]");
+      await page.getByTestId("conversation-form").waitFor();
+      await page.click("[data-testid=conversation-form-submit]");
+
+      // wait for the conversation to be created
+      await page.getByTestId("conversation-page").waitFor();
+
+      // submit a message to the conversation
+      await page.getByTestId("conversation-page-input").fill("How are you?");
+      await page.getByTestId("conversation-page-submit").click();
+      await page.locator(".ai-message").waitFor();
+      const player = page
+        .locator(".ai-message")
+        .getByTestId("wavesurfer-container");
+      await player.waitFor();
+
+      expect(await player.isVisible()).toBeTruthy();
     });
 
     /*
@@ -136,6 +166,9 @@ test.describe("with login", async () => {
      * 5. audio waveform player should be visible and transcription should be generated
      */
     test("gpt conversation", async () => {
+      // navigate to the conversations page
+      await page.getByTestId("sidebar-conversations").click();
+
       // trigger new conversation modal
       await page.getByTestId("conversation-new-button").click();
 
@@ -166,43 +199,12 @@ test.describe("with login", async () => {
 
       // add to library
       await page.getByTestId("message-start-shadow").click();
-      await page.getByTestId("audio-detail").waitFor();
+      await page.getByTestId("audio-player").waitFor();
       await page.getByTestId("media-player-container").waitFor();
-      await page.getByTestId("media-transcription").waitFor();
       await page.getByTestId("media-transcription-result").waitFor();
       expect(
         await page.getByTestId("media-transcription-result").isVisible()
       ).toBeTruthy();
     });
-
-    /*
-     * steps:
-     * 1. create a tts conversation
-     * 2. submit a message to the conversation
-     * 3. the speech should auto create
-     */
-    test("tts conversation", async () => {
-      // trigger new conversation modal
-      await page.getByTestId("conversation-new-button").click();
-
-      // create a tts conversation
-      await page.click("[data-testid=conversation-preset-tts]");
-      await page.getByTestId("conversation-form").waitFor();
-      await page.click("[data-testid=conversation-form-submit]");
-
-      // wait for the conversation to be created
-      await page.getByTestId("conversation-page").waitFor();
-
-      // submit a message to the conversation
-      await page.getByTestId("conversation-page-input").fill("How are you?");
-      await page.getByTestId("conversation-page-submit").click();
-      await page.locator(".ai-message").waitFor();
-      const player = page
-        .locator(".ai-message")
-        .getByTestId("wavesurfer-container");
-      await player.waitFor();
-
-      expect(await player.isVisible()).toBeTruthy();
-    });
   });
 });
diff --git a/enjoy/forge.config.js b/enjoy/forge.config.js
@@ -12,7 +12,7 @@ const config = {
     asar: {
       // Binary files won't work in asar, so we need to unpack them
       unpackDir:
-        "{.vite/build/lib,.vite/build/samples,node_modules/ffmpeg-static,node_modules/@andrkrn/ffprobe-static}",
+        "{.vite/build/lib,.vite/build/samples,node_modules/ffmpeg-static,node_modules/@andrkrn/ffprobe-static,node_modules/onnxruntime-node/bin}",
     },
     icon: "./assets/icon",
     name: "Enjoy",

diff --git a/enjoy/package.json b/enjoy/package.json
@@ -47,18 +47,18 @@
     "@types/fluent-ffmpeg": "^2.1.24",
     "@types/html-to-text": "^9.0.4",
     "@types/intl-tel-input": "^18.1.4",
-    "@types/lodash": "^4.14.202",
+    "@types/lodash": "^4.17.0",
     "@types/mark.js": "^8.11.12",
-    "@types/node": "^20.11.24",
-    "@types/react": "^18.2.62",
-    "@types/react-dom": "^18.2.19",
+    "@types/node": "^20.11.27",
+    "@types/react": "^18.2.66",
+    "@types/react-dom": "^18.2.22",
     "@types/validator": "^13.11.9",
     "@types/wavesurfer.js": "^6.0.12",
-    "@typescript-eslint/eslint-plugin": "^7.1.1",
-    "@typescript-eslint/parser": "^7.1.1",
+    "@typescript-eslint/eslint-plugin": "^7.2.0",
+    "@typescript-eslint/parser": "^7.2.0",
     "@vitejs/plugin-react": "^4.2.1",
     "autoprefixer": "^10.4.18",
-    "electron": "^29.1.0",
+    "electron": "^29.1.4",
     "electron-playwright-helpers": "^1.7.1",
     "eslint": "^8.57.0",
     "eslint-import-resolver-typescript": "^3.6.1",
@@ -67,12 +67,13 @@
     "octokit": "^3.1.2",
     "progress": "^2.0.3",
     "tailwind-merge": "^2.2.1",
+    "tailwind-scrollbar": "^3.1.0",
     "tailwindcss": "^3.4.1",
     "tailwindcss-animate": "^1.0.7",
     "ts-node": "^10.9.2",
     "tslib": "^2.6.2",
-    "typescript": "^5.3.3",
-    "vite": "^5.1.5",
+    "typescript": "^5.4.2",
+    "vite": "^5.1.6",
     "vite-plugin-static-copy": "^1.0.1",
     "zx": "^7.2.3"
   },
@@ -81,7 +82,7 @@
     "@ffmpeg/ffmpeg": "^0.12.10",
     "@ffmpeg/util": "^0.12.1",
     "@hookform/resolvers": "^3.3.4",
-    "@langchain/community": "^0.0.34",
+    "@langchain/community": "^0.0.39",
     "@langchain/google-genai": "^0.0.10",
     "@mozilla/readability": "^0.5.0",
     "@radix-ui/react-accordion": "^1.1.2",
@@ -112,6 +113,7 @@
     "axios": "^1.6.7",
     "camelcase": "^8.0.0",
     "camelcase-keys": "^9.1.3",
+    "chart.js": "^4.4.2",
     "cheerio": "^1.0.0-rc.12",
     "class-variance-authority": "^0.7.0",
     "clsx": "^2.1.0",
@@ -122,35 +124,36 @@
     "dayjs": "^1.11.10",
     "decamelize": "^6.0.0",
     "decamelize-keys": "^2.0.1",
-    "electron-log": "^5.1.1",
+    "echogarden": "https://github.com/an-lee/echogarden",
+    "electron-log": "^5.1.2",
     "electron-settings": "^4.0.2",
     "electron-squirrel-startup": "^1.0.0",
     "ffmpeg-static": "^5.2.0",
     "fluent-ffmpeg": "^2.1.2",
     "fs-extra": "^11.2.0",
     "html-to-text": "^9.0.5",
     "https-proxy-agent": "^7.0.4",
-    "i18next": "^23.10.0",
-    "intl-tel-input": "^19.5.5",
+    "i18next": "^23.10.1",
+    "intl-tel-input": "^19.5.7",
     "js-md5": "^0.8.3",
-    "langchain": "^0.1.25",
+    "langchain": "^0.1.28",
     "lodash": "^4.17.21",
-    "lucide-react": "^0.344.0",
+    "lucide-react": "^0.358.0",
     "mark.js": "^8.11.1",
-    "microsoft-cognitiveservices-speech-sdk": "^1.35.0",
-    "next-themes": "^0.2.1",
-    "openai": "^4.28.4",
+    "microsoft-cognitiveservices-speech-sdk": "^1.36.0",
+    "next-themes": "^0.3.0",
+    "openai": "^4.29.0",
     "pitchfinder": "^2.3.2",
     "postcss": "^8.4.35",
     "proxy-agent": "^6.4.0",
     "react": "^18.2.0",
-    "react-activity-calendar": "^2.2.7",
+    "react-activity-calendar": "^2.2.8",
     "react-dom": "^18.2.0",
     "react-hook-form": "^7.51.0",
     "react-hotkeys-hook": "^4.5.0",
-    "react-i18next": "^14.0.5",
+    "react-i18next": "^14.1.0",
     "react-markdown": "^9.0.1",
-    "react-router-dom": "^6.22.2",
+    "react-router-dom": "^6.22.3",
     "react-tooltip": "^5.26.3",
     "reflect-metadata": "^0.2.1",
     "rimraf": "^5.0.5",
@@ -160,7 +163,7 @@
     "sqlite3": "^5.1.7",
     "tailwind-scrollbar-hide": "^1.1.7",
     "umzug": "^3.7.0",
-    "wavesurfer.js": "^7.7.3",
+    "wavesurfer.js": "^7.7.5",
     "zod": "^3.22.4"
   }
 }
diff --git a/enjoy/playwright.config.ts b/enjoy/playwright.config.ts
@@ -18,7 +18,7 @@ export default defineConfig({
   /* Retry on CI only */
   retries: process.env.CI ? 2 : 0,
   /* Opt out of parallel tests on CI. */
-  workers: process.env.CI ? 1 : undefined,
+  workers: 1,
   /* Reporter to use. See https://playwright.dev/docs/test-reporters */
   reporter: "html",
   /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */