fix: Audio tool split and merge text by clip

open-mmlab · May 14, 2024 · fb5d559 · fb5d559
1 parent 3b22374
commit fb5d559
Show file tree

Hide file tree

Showing 4 changed files with 73 additions and 27 deletions.
diff --git a/packages/lb-components/src/components/audioAnnotate/utils/dataTransform.ts b/packages/lb-components/src/components/audioAnnotate/utils/dataTransform.ts
@@ -0,0 +1,28 @@
+/**
+ * @file Process audio tool data
+ * @author lixinghua <lixinghua_vendor@sensetime.com>
+ * @date 2024.05.14
+ */
+
+import { IAudioTimeSlice, ITextConfigItem } from '@labelbee/lb-utils';
+import _ from 'lodash';
+
+export default class DataTransform {
+  // clip tool get text by config
+  public static getClipTextByConfig = (
+    region: IAudioTimeSlice,
+    clipTextList: ITextConfigItem[],
+    isClear = false,
+  ) => {
+    const newRegion = _.cloneDeep(region);
+    clipTextList.forEach((i, index) => {
+      // index === 0: Compatible with old data
+      if (index === 0) {
+        Object.assign(newRegion, { text: isClear ? '' : region[i.key] });
+      } else {
+        Object.assign(newRegion, { [i.key]: isClear ? '' : region[i.key] });
+      }
+    });
+    return newRegion;
+  };
+}
diff --git a/packages/lb-components/src/components/audioPlayer/index.tsx b/packages/lb-components/src/components/audioPlayer/index.tsx
@@ -3,7 +3,7 @@ import { getWebPcm2WavBase64 } from '@/components/audioAnnotate/utils/getWebPcm2
 import _, { debounce, sortBy } from 'lodash';
 import { PauseOutlined, CaretRightOutlined } from '@ant-design/icons';
 import { cKeyCode, cTool, EventBus, TagUtils } from '@labelbee/lb-annotation';
-import { IAudioTimeSlice,ITextConfigItem } from '@labelbee/lb-utils';
+import { IAudioTimeSlice, ITextConfigItem } from '@labelbee/lb-utils';
 import { Button } from 'antd';
 import InvalidPage from '@/components/invalidPage';
 import ImageError from '@/components/imageError';
@@ -245,7 +245,7 @@ export const AudioPlayer = ({
     clipConfigurable,
     secondaryAttributeConfigurable,
     subAttributeList,
-    clipTextList
+    clipTextList,
   };
 
   useEffect(() => {

diff --git a/packages/lb-components/src/components/audioPlayer/useAudioCombine/index.tsx b/packages/lb-components/src/components/audioPlayer/useAudioCombine/index.tsx
@@ -9,10 +9,11 @@ import { cKeyCode } from '@labelbee/lb-annotation';
 import { useAudioClipStore } from '@/components/audioAnnotate/audioContext';
 import { useEventListener, useMemoizedFn } from 'ahooks';
 import { useEffect } from 'react';
-import { IAudioTimeSlice } from '@labelbee/lb-utils'
+import { IAudioTimeSlice } from '@labelbee/lb-utils';
 import { ISetSelectedRegionParams } from '..';
+import { cloneDeep } from 'lodash';
 
-const EKeyCode = cKeyCode.default
+const EKeyCode = cKeyCode.default;
 
 interface IProps {
   /** WaveSurfer */
@@ -43,9 +44,34 @@ const useAudioCombine = (props: IProps) => {
     setSelectedRegion,
   } = props;
   const { audioClipState, setAudioClipState } = useAudioClipStore();
-  const { selectedRegion, clipConfigurable, combined } = audioClipState;
+  const { selectedRegion, clipConfigurable, combined, clipTextList } = audioClipState;
   const { id } = selectedRegion;
 
+  const combineTextByConfig = (
+    region: IAudioTimeSlice,
+    current: IAudioTimeSlice,
+    target: IAudioTimeSlice,
+  ) => {
+    const newRegion = cloneDeep(region);
+    // If the merged text is not empty, it needs to be wrapped.
+    clipTextList.forEach((i, index) => {
+      const curText = current[i.key];
+      const targetText = target[i.key];
+      const inCludeEmpty = [curText, targetText].includes('');
+      const showText = inCludeEmpty
+        ? `${curText}${targetText}`
+        : `${curText}
+${targetText}`;
+      // index === 0: Compatible with old data
+      if (index === 0) {
+        Object.assign(newRegion, { text: showText });
+      } else {
+        Object.assign(newRegion, { [i.key]: showText });
+      }
+    });
+    return newRegion;
+  };
+
   const combineInstance = useMemoizedFn((instance) => {
     if (!id) {
       return;
@@ -75,7 +101,7 @@ const useAudioCombine = (props: IProps) => {
     const start = Math.min(...times);
     const end = Math.max(...times);
 
-    const newRegion = {
+    const region = {
       id: waveRef.current?.util.getId('combined_'),
       start,
       end,
@@ -84,8 +110,10 @@ const useAudioCombine = (props: IProps) => {
       text: [current.text, target.text].includes('')
         ? `${current.text}${target.text}`
         : `${current.text}
-${target.text}`,
+        ${target.text}`,
     };
+    const newRegion = combineTextByConfig(region, current, target);
+
     updateRegion?.(newRegion);
     removeRegion?.(id);
     removeRegion?.(target.id);

diff --git a/packages/lb-components/src/components/audioPlayer/useAudioSegment/index.tsx b/packages/lb-components/src/components/audioPlayer/useAudioSegment/index.tsx
@@ -10,9 +10,9 @@ import { cKeyCode } from '@labelbee/lb-annotation';
 import { useAudioClipStore } from '@/components/audioAnnotate/audioContext';
 import { useEventListener, useMemoizedFn } from 'ahooks';
 import { ISetSelectedRegionParams } from '..';
-import { IAudioTimeSlice } from '@labelbee/lb-utils'
-
-const EKeyCode = cKeyCode.default
+import { IAudioTimeSlice } from '@labelbee/lb-utils';
+import DataTransform from '@/components/audioAnnotate/utils/dataTransform';
+const EKeyCode = cKeyCode.default;
 
 interface IProps {
   /** WaveSurfer */
@@ -32,17 +32,11 @@ interface IProps {
 
 /** 音频区间分割功能 */
 const useAudioSegment = (props: IProps) => {
-  const {
-    waveRef,
-    regionMap,
-    updateRegion,
-    removeRegion,
-    generateRegions,
-    setSelectedRegion,
-  } = props;
+  const { waveRef, regionMap, updateRegion, removeRegion, generateRegions, setSelectedRegion } =
+    props;
 
   const { audioClipState, setAudioClipState } = useAudioClipStore();
-  const { selectedRegion, clipConfigurable, segment } = audioClipState;
+  const { selectedRegion, clipConfigurable, segment, clipTextList } = audioClipState;
   const { id } = selectedRegion;
   const segmentTimeTip = useRef<null | number>(null);
   const mouseEvent = useRef<null | MouseEvent>(null);
@@ -68,21 +62,17 @@ const useAudioSegment = (props: IProps) => {
       return;
     }
     const current = regionMap[id];
-
+    const newData = DataTransform.getClipTextByConfig(current, clipTextList);
     const targetLeft = {
+      ...newData,
       id: waveRef.current?.util.getId('segment_'),
-      start: current.start,
       end: time,
-      attribute: current.attribute,
-      text: current.text,
     };
-
+    const clearText = DataTransform.getClipTextByConfig(current, clipTextList, true);
     const targetRight = {
+      ...clearText,
       id: waveRef.current?.util.getId('segment_'),
       start: time,
-      end: current.end,
-      attribute: current.attribute,
-      text: '',
     };
 
     updateRegion?.(targetLeft);