feat: async voice messages recording

package.json

@@ -66,12 +66,14 @@
     "clsx": "^2.0.0",
     "dayjs": "^1.10.4",
     "emoji-regex": "^9.2.0",
+    "fix-webm-duration": "^1.0.5",
     "hast-util-find-and-replace": "^5.0.1",
     "i18next": "^21.6.14",
     "isomorphic-ws": "^4.0.1",
     "linkifyjs": "^4.1.0",
     "lodash.debounce": "^4.0.8",
     "lodash.defaultsdeep": "^4.6.1",
+    "lodash.mergewith": "^4.6.2",
     "lodash.throttle": "^4.1.1",
     "lodash.uniqby": "^4.7.0",
     "nanoid": "^3.3.4",
@@ -157,6 +159,7 @@
     "@types/linkifyjs": "^2.1.3",
     "@types/lodash.debounce": "^4.0.7",
     "@types/lodash.defaultsdeep": "^4.6.9",
+    "@types/lodash.mergewith": "^4.6.9",
     "@types/lodash.throttle": "^4.1.7",
     "@types/lodash.uniqby": "^4.7.7",
     "@types/moment": "^2.13.0",

src/components/Attachment/VoiceRecording.tsx

@@ -22,7 +22,7 @@ export const VoiceRecordingPlayer = ({ attachment, playbackRates }: VoiceRecordi
   const { t } = useTranslationContext('VoiceRecordingPlayer');
   const {
     asset_url,
-    duration,
+    duration = 0,
     mime_type,
     title = t<string>('Voice message'),
     waveform_data,
@@ -39,11 +39,14 @@ export const VoiceRecordingPlayer = ({ attachment, playbackRates }: VoiceRecordi
     togglePlay,
   } = useAudioController({
     durationSeconds: duration ?? 0,
+    mimeType: mime_type,
     playbackRates,
   });
 
   if (!asset_url) return null;
 
+  const displayedDuration = secondsElapsed || duration;
+
   return (
     <div className={rootClassName} data-testid='voice-recording-widget'>
       <audio ref={audioRef}>
@@ -61,7 +64,7 @@ export const VoiceRecordingPlayer = ({ attachment, playbackRates }: VoiceRecordi
         <div className='str-chat__message-attachment__voice-recording-widget__audio-state'>
           <div className='str-chat__message-attachment__voice-recording-widget__timer'>
             {attachment.duration ? (
-              displayDuration(secondsElapsed)
+              displayDuration(displayedDuration)
             ) : (
               <FileSizeIndicator fileSize={attachment.file_size} maximumFractionDigits={0} />
             )}

src/components/Attachment/__tests__/WaveProgressBar.test.js

@@ -1,7 +1,8 @@
 import React from 'react';
 import { render, screen } from '@testing-library/react';
 import '@testing-library/jest-dom';
-import { downSample, upSample, WaveProgressBar } from '../components';
+import { WaveProgressBar } from '../components';
+import { downSample, upSample } from '../audioSampling';
 
 jest.spyOn(console, 'warn').mockImplementation();
 const originalSample = Array.from({ length: 10 }, (_, i) => i);

src/components/Attachment/audioSampling.ts

@@ -0,0 +1,106 @@
+import { divMod } from './utils';
+
+export const resampleWaveformData = (waveformData: number[], amplitudesCount: number) =>
+  waveformData.length === amplitudesCount
+    ? waveformData
+    : waveformData.length > amplitudesCount
+    ? downSample(waveformData, amplitudesCount)
+    : upSample(waveformData, amplitudesCount);
+
+/**
+ * The downSample function uses the Largest-Triangle-Three-Buckets (LTTB) algorithm.
+ * See the thesis Downsampling Time Series for Visual Representation by Sveinn Steinarsson for more (https://skemman.is/bitstream/1946/15343/3/SS_MSthesis.pdf)
+ * @param data
+ * @param targetOutputSize
+ */
+export function downSample(data: number[], targetOutputSize: number): number[] {
+  if (data.length <= targetOutputSize || targetOutputSize === 0) {
+    return data;
+  }
+
+  if (targetOutputSize === 1) return [mean(data)];
+
+  const result: number[] = [];
+  // bucket size adjusted due to the fact that the first and the last item in the original data array is kept in target output
+  const bucketSize = (data.length - 2) / (targetOutputSize - 2);
+  let lastSelectedPointIndex = 0;
+  result.push(data[lastSelectedPointIndex]); // Always add the first point
+  let maxAreaPoint, maxArea, triangleArea;
+
+  for (let bucketIndex = 1; bucketIndex < targetOutputSize - 1; bucketIndex++) {
+    const previousBucketRefPoint = data[lastSelectedPointIndex];
+    const nextBucketMean = getNextBucketMean(data, bucketIndex, bucketSize);
+
+    const currentBucketStartIndex = Math.floor((bucketIndex - 1) * bucketSize) + 1;
+    const nextBucketStartIndex = Math.floor(bucketIndex * bucketSize) + 1;
+    const countUnitsBetweenAtoC = 1 + nextBucketStartIndex - currentBucketStartIndex;
+
+    maxArea = triangleArea = -1;
+
+    for (
+      let currentPointIndex = currentBucketStartIndex;
+      currentPointIndex < nextBucketStartIndex;
+      currentPointIndex++
+    ) {
+      const countUnitsBetweenAtoB = Math.abs(currentPointIndex - currentBucketStartIndex) + 1;
+      const countUnitsBetweenBtoC = countUnitsBetweenAtoC - countUnitsBetweenAtoB;
+      const currentPointValue = data[currentPointIndex];
+
+      triangleArea = triangleAreaHeron(
+        triangleBase(Math.abs(previousBucketRefPoint - currentPointValue), countUnitsBetweenAtoB),
+        triangleBase(Math.abs(currentPointValue - nextBucketMean), countUnitsBetweenBtoC),
+        triangleBase(Math.abs(previousBucketRefPoint - nextBucketMean), countUnitsBetweenAtoC),
+      );
+
+      if (triangleArea > maxArea) {
+        maxArea = triangleArea;
+        maxAreaPoint = data[currentPointIndex];
+        lastSelectedPointIndex = currentPointIndex;
+      }
+    }
+
+    if (typeof maxAreaPoint !== 'undefined') result.push(maxAreaPoint);
+  }
+
+  result.push(data[data.length - 1]); // Always add the last point
+
+  return result;
+}
+
+const triangleAreaHeron = (a: number, b: number, c: number) => {
+  const s = (a + b + c) / 2;
+  return Math.sqrt(s * (s - a) * (s - b) * (s - c));
+};
+const triangleBase = (a: number, b: number) => Math.sqrt(Math.pow(a, 2) + Math.pow(b, 2));
+const mean = (values: number[]) => values.reduce((acc, value) => acc + value, 0) / values.length;
+const getNextBucketMean = (data: number[], currentBucketIndex: number, bucketSize: number) => {
+  const nextBucketStartIndex = Math.floor(currentBucketIndex * bucketSize) + 1;
+  let nextNextBucketStartIndex = Math.floor((currentBucketIndex + 1) * bucketSize) + 1;
+  nextNextBucketStartIndex =
+    nextNextBucketStartIndex < data.length ? nextNextBucketStartIndex : data.length;
+
+  return mean(data.slice(nextBucketStartIndex, nextNextBucketStartIndex));
+};
+export const upSample = (values: number[], targetSize: number) => {
+  if (!values.length) {
+    console.warn('Cannot extend empty array of amplitudes.');
+    return values;
+  }
+
+  if (values.length > targetSize) {
+    console.warn('Requested to extend the waveformData that is longer than the target list size');
+    return values;
+  }
+
+  if (targetSize === values.length) return values;
+
+  // eslint-disable-next-line prefer-const
+  let [bucketSize, remainder] = divMod(targetSize, values.length);
+  const result: number[] = [];
+
+  for (let i = 0; i < values.length; i++) {
+    const extra = remainder && remainder-- ? 1 : 0;
+    result.push(...Array(bucketSize + extra).fill(values[i]));
+  }
+  return result;
+};

src/components/Attachment/components/WaveProgressBar.tsx

@@ -1,17 +1,27 @@
-import React, { MouseEventHandler, useMemo, useRef, useState } from 'react';
+import React, {
+  MouseEvent,
+  MouseEventHandler,
+  TouchEvent,
+  TouchEventHandler,
+  useMemo,
+  useRef,
+  useState,
+} from 'react';
 import clsx from 'clsx';
-import { divMod } from '../utils';
+import { resampleWaveformData } from '../audioSampling';
+import type { SeekFn } from '../hooks/useAudioController';
 
 type WaveProgressBarProps = {
   /** Function that allows to change the track progress */
-  seek: MouseEventHandler<HTMLDivElement>;
+  seek: SeekFn;
   /** The array of fractional number values between 0 and 1 representing the height of amplitudes */
   waveformData: number[];
   /** Allows to specify the number of bars into which the original waveformData array should be resampled */
   amplitudesCount?: number;
   /** Progress expressed in fractional number value btw 0 and 100. */
   progress?: number;
 };
+
 export const WaveProgressBar = ({
   amplitudesCount = 40,
   progress = 0,
@@ -20,16 +30,25 @@ export const WaveProgressBar = ({
 }: WaveProgressBarProps) => {
   const [progressIndicator, setProgressIndicator] = useState<HTMLDivElement | null>(null);
   const isDragging = useRef(false);
+  const rootRef = useRef<HTMLDivElement | null>(null);
 
-  const handleMouseDown = () => {
+  const handleDragStart = (e: MouseEvent | TouchEvent) => {
+    e.preventDefault();
     if (!progressIndicator) return;
     isDragging.current = true;
     progressIndicator.style.cursor = 'grabbing';
   };
 
   const handleMouseMove: MouseEventHandler<HTMLDivElement> = (event) => {
     if (!isDragging.current) return;
-    seek(event);
+    seek({ ...event });
+  };
+
+  const handleTouchMove: TouchEventHandler<HTMLDivElement> = (event) => {
+    if (!(rootRef.current && isDragging.current)) return;
+    event.preventDefault();
+    const touch = event.touches[0];
+    seek({ clientX: touch.clientX, currentTarget: rootRef.current });
   };
 
   const handleMouseUp = () => {
@@ -38,15 +57,10 @@ export const WaveProgressBar = ({
     progressIndicator.style.removeProperty('cursor');
   };
 
-  const resampledWaveformData = useMemo(
-    () =>
-      waveformData.length === amplitudesCount
-        ? waveformData
-        : waveformData.length > amplitudesCount
-        ? downSample(waveformData, amplitudesCount)
-        : upSample(waveformData, amplitudesCount),
-    [amplitudesCount, waveformData],
-  );
+  const resampledWaveformData = useMemo(() => resampleWaveformData(waveformData, amplitudesCount), [
+    amplitudesCount,
+    waveformData,
+  ]);
 
   if (!waveformData.length) return null;
 
@@ -55,10 +69,14 @@ export const WaveProgressBar = ({
       className='str-chat__wave-progress-bar__track'
       data-testid='wave-progress-bar-track'
       onClick={seek}
-      onMouseDown={handleMouseDown}
+      onMouseDown={handleDragStart}
       onMouseLeave={handleMouseUp}
       onMouseMove={handleMouseMove}
       onMouseUp={handleMouseUp}
+      onTouchEnd={handleMouseUp}
+      onTouchMove={handleTouchMove}
+      onTouchStart={handleDragStart}
+      ref={rootRef}
       role='progressbar'
     >
       {resampledWaveformData.map((amplitude, i) => (
@@ -87,105 +105,3 @@ export const WaveProgressBar = ({
     </div>
   );
 };
-
-/**
- * The downSample function uses the Largest-Triangle-Three-Buckets (LTTB) algorithm.
- * See the thesis Downsampling Time Series for Visual Representation by Sveinn Steinarsson for more (https://skemman.is/bitstream/1946/15343/3/SS_MSthesis.pdf)
- * @param data
- * @param targetOutputSize
- */
-export function downSample(data: number[], targetOutputSize: number): number[] {
-  if (data.length <= targetOutputSize || targetOutputSize === 0) {
-    return data;
-  }
-
-  if (targetOutputSize === 1) return [mean(data)];
-
-  const result: number[] = [];
-  // bucket size adjusted due to the fact that the first and the last item in the original data array is kept in target output
-  const bucketSize = (data.length - 2) / (targetOutputSize - 2);
-  let lastSelectedPointIndex = 0;
-  result.push(data[lastSelectedPointIndex]); // Always add the first point
-  let maxAreaPoint, maxArea, triangleArea;
-
-  for (let bucketIndex = 1; bucketIndex < targetOutputSize - 1; bucketIndex++) {
-    const previousBucketRefPoint = data[lastSelectedPointIndex];
-    const nextBucketMean = getNextBucketMean(data, bucketIndex, bucketSize);
-
-    const currentBucketStartIndex = Math.floor((bucketIndex - 1) * bucketSize) + 1;
-    const nextBucketStartIndex = Math.floor(bucketIndex * bucketSize) + 1;
-    const countUnitsBetweenAtoC = 1 + nextBucketStartIndex - currentBucketStartIndex;
-
-    maxArea = triangleArea = -1;
-
-    for (
-      let currentPointIndex = currentBucketStartIndex;
-      currentPointIndex < nextBucketStartIndex;
-      currentPointIndex++
-    ) {
-      const countUnitsBetweenAtoB = Math.abs(currentPointIndex - currentBucketStartIndex) + 1;
-      const countUnitsBetweenBtoC = countUnitsBetweenAtoC - countUnitsBetweenAtoB;
-      const currentPointValue = data[currentPointIndex];
-
-      triangleArea = triangleAreaHeron(
-        triangleBase(Math.abs(previousBucketRefPoint - currentPointValue), countUnitsBetweenAtoB),
-        triangleBase(Math.abs(currentPointValue - nextBucketMean), countUnitsBetweenBtoC),
-        triangleBase(Math.abs(previousBucketRefPoint - nextBucketMean), countUnitsBetweenAtoC),
-      );
-
-      if (triangleArea > maxArea) {
-        maxArea = triangleArea;
-        maxAreaPoint = data[currentPointIndex];
-        lastSelectedPointIndex = currentPointIndex;
-      }
-    }
-
-    if (typeof maxAreaPoint !== 'undefined') result.push(maxAreaPoint);
-  }
-
-  result.push(data[data.length - 1]); // Always add the last point
-
-  return result;
-}
-
-const triangleAreaHeron = (a: number, b: number, c: number) => {
-  const s = (a + b + c) / 2;
-  return Math.sqrt(s * (s - a) * (s - b) * (s - c));
-};
-
-const triangleBase = (a: number, b: number) => Math.sqrt(Math.pow(a, 2) + Math.pow(b, 2));
-
-const mean = (values: number[]) => values.reduce((acc, value) => acc + value, 0) / values.length;
-
-const getNextBucketMean = (data: number[], currentBucketIndex: number, bucketSize: number) => {
-  const nextBucketStartIndex = Math.floor(currentBucketIndex * bucketSize) + 1;
-  let nextNextBucketStartIndex = Math.floor((currentBucketIndex + 1) * bucketSize) + 1;
-  nextNextBucketStartIndex =
-    nextNextBucketStartIndex < data.length ? nextNextBucketStartIndex : data.length;
-
-  return mean(data.slice(nextBucketStartIndex, nextNextBucketStartIndex));
-};
-
-export const upSample = (values: number[], targetSize: number) => {
-  if (!values.length) {
-    console.warn('Cannot extend empty array of amplitudes.');
-    return values;
-  }
-
-  if (values.length > targetSize) {
-    console.warn('Requested to extend the waveformData that is longer than the target list size');
-    return values;
-  }
-
-  if (targetSize === values.length) return values;
-
-  // eslint-disable-next-line prefer-const
-  let [bucketSize, remainder] = divMod(targetSize, values.length);
-  const result: number[] = [];
-
-  for (let i = 0; i < values.length; i++) {
-    const extra = remainder && remainder-- ? 1 : 0;
-    result.push(...Array(bucketSize + extra).fill(values[i]));
-  }
-  return result;
-};