[Feature] Transcriptions and translations (#994)

* Transcription text component * Event just from react * refactor position function * wip transcription with history * margin last child * add translation * fix tests * remove unused import * WIP transcription history * Hide transcript participant * Change transcription history * wip transcription * Add default jigasi gc values * change copy * Wip translation * adding stop translating * Position button * Drilling refactor and enable transcription drawer * Transcription enabled in backend with env * Fixing reviews, selector, name and only if transcr enabled * set language at connection and fix coded characters * Fix enable translating too fast * Add flag check from env * wip * hide history w/o messages env flag * change audio transcriber * Fix function call * wip * Add queue and scroll * Wip select language * WIP Loading * WIP UI Updated * WIP UI * WIP UI cookie * Unused variables * temporary show transcriber user * Remove temporary participant * change env * Reviews UI and modal * review * Fix typescript * fix position * Fix select, position and toggle * added state * logo in toggle and spacing * wip ui fix * fix empty caption * fix spacings and color * Copies * language selector localized * fix translation errors * Change jitsi nginx colibri * change jitsi ngnix config for colibri * fix mobile * improve mobile experience * Unused import * order selector * width title * fix translation * Add jigasi logs * Disable transcriptions by default * Disabled backend transcriptions * enable jigasi transcription * Fix token transcription boolean * Fix transcription selector --------- Co-authored-by: Aitor Errasti <azarparalelo@gmail.com>
Stooa · Jun 28, 2023 · 9789bcb · 9789bcb
1 parent 0b60295
commit 9789bcb
Show file tree

Hide file tree

Showing 72 changed files with 1,794 additions and 221 deletions.
diff --git a/.env b/.env
@@ -55,6 +55,27 @@ ENABLE_CODEC_VP9=true
 ENABLE_CODEC_H264=true
 JICOFO_ENABLE_HEALTH_CHECKS=true
 
+# JIGASI
+JIGASI_DOMAIN=jigasi.meet.jitsi
+JIGASI_SIP_URI=test@sip2sip.info
+JIGASI_SIP_PASSWORD=password
+JIGASI_SIP_SERVER=sip2sip.info
+JIGASI_SIP_PORT=5060
+JIGASI_SIP_TRANSPORT=UDP
+JIGASI_XMPP_PASSWORD=password
+ENABLE_TRANSCRIPTIONS=1
+JIGASI_TRANSCRIBER_SEND_TXT=1
+JIGASI_TRANSCRIBER_RECORD_AUDIO=0
+JIGASI_TRANSCRIBER_ADVERTISE_URL=0
+
+# Google Cloud credentials
+GC_PROJECT_ID=project_id
+GC_PRIVATE_KEY_ID=private_key_id
+GC_PRIVATE_KEY=private_key
+GC_CLIENT_EMAIL=client_email
+GC_CLIENT_ID=client_id
+GC_CLIENT_CERT_URL=client_cert_url
+
 # JWT AUTH
 JWT_APP_ID=api_client
 JWT_APP_SECRET=this_is_a_secret_key_for_stooa_platform

diff --git a/backend/.env b/backend/.env
@@ -29,3 +29,5 @@ JWT_PASSPHRASE=pass_phrase
 JAAS_ACTIVE=false
 JAAS_APP_ID=
 JAAS_API_KEY=
+
+TRANSCRIPTION_ENABLED=false
diff --git a/backend/packages/Core/config/services.yaml b/backend/packages/Core/config/services.yaml
@@ -40,6 +40,7 @@ services:
         bind:
             $appId: '%env(JAAS_APP_ID)%'
             $apiKey: '%env(JAAS_API_KEY)%'
+            $transcriptionEnabled: '%env(bool:TRANSCRIPTION_ENABLED)%'
 
     App\Core\Encryption\HalitePasswordEncryption:
         bind:

diff --git a/backend/packages/Core/src/JWT/TokenGenerator/JaasTokenGenerator.php b/backend/packages/Core/src/JWT/TokenGenerator/JaasTokenGenerator.php
@@ -25,6 +25,7 @@ final class JaasTokenGenerator implements TokenGeneratorInterface
     public function __construct(
         private readonly string $appId,
         private readonly string $apiKey,
+        private readonly bool $transcriptionEnabled,
         private readonly HostValidator $hostValidator
     ) {
     }
@@ -36,7 +37,7 @@ public function generate(User $user): JWTToken
         return new JWTToken('chat', 'jitsi', $this->appId, '*', $userPayload,
             new \DateTimeImmutable('-10 seconds'),
             new HeaderPayload($this->apiKey, 'RS256', 'JWT'),
-            new FeaturesPayload(false, false, false, false, false)
+            new FeaturesPayload(false, false, $this->transcriptionEnabled, false, false)
         );
     }
 }
diff --git a/backend/packages/Core/tests/Unit/JWT/JaasTokenGeneratorTest.php b/backend/packages/Core/tests/Unit/JWT/JaasTokenGeneratorTest.php
@@ -35,19 +35,26 @@ class JaasTokenGeneratorTest extends TestCase
     private User $user;
     private string $appId;
     private string $apiKey;
+    private bool $transcriptionEnabled;
 
     protected function setUp(): void
     {
         parent::setUp();
 
         $this->appId = 'appId';
         $this->apiKey = 'apiKey';
+        $this->transcriptionEnabled = false;
 
         $this->user = UserFactory::createOne()->object();
 
         $this->hostValidator = $this->createStub(HostValidator::class);
 
-        $this->jaasTokenGenerator = new JaasTokenGenerator($this->appId, $this->apiKey, $this->hostValidator);
+        $this->jaasTokenGenerator = new JaasTokenGenerator(
+            $this->appId,
+            $this->apiKey,
+            $this->transcriptionEnabled,
+            $this->hostValidator
+        );
     }
 
     /** @test */

diff --git a/docker-compose.yaml b/docker-compose.yaml
@@ -270,6 +270,60 @@ services:
             meet.jitsi:
                 aliases:
                     - ${JVB_SERVER}
+    jigasi:
+        image: jitsi/jigasi:stable-8319
+        restart: ${RESTART_POLICY:-unless-stopped}
+        ports:
+            - '${JIGASI_PORT_MIN:-20000}-${JIGASI_PORT_MAX:-20050}:${JIGASI_PORT_MIN:-20000}-${JIGASI_PORT_MAX:-20050}/udp'
+        volumes:
+            - ${CONFIG}/jigasi:/config:Z
+            - ${CONFIG}/transcripts:/tmp/transcripts:Z
+            - ./jitsi/jigasi/custom-sip-communicator.properties:/config/custom-sip-communicator.properties:Z
+        environment:
+            - ENABLE_AUTH
+            - ENABLE_GUESTS
+            - XMPP_AUTH_DOMAIN
+            - XMPP_GUEST_DOMAIN
+            - XMPP_MUC_DOMAIN
+            - XMPP_INTERNAL_MUC_DOMAIN
+            - XMPP_SERVER
+            - XMPP_PORT
+            - XMPP_DOMAIN
+            - PUBLIC_URL
+            - JIGASI_DISABLE_SIP
+            - JIGASI_SIP_URI
+            - JIGASI_SIP_PASSWORD
+            - JIGASI_SIP_SERVER
+            - JIGASI_SIP_PORT
+            - JIGASI_SIP_TRANSPORT
+            - JIGASI_SIP_DEFAULT_ROOM
+            - JIGASI_XMPP_USER
+            - JIGASI_XMPP_PASSWORD
+            - JIGASI_BREWERY_MUC
+            - JIGASI_PORT_MIN
+            - JIGASI_PORT_MAX
+            - JIGASI_HEALTH_CHECK_SIP_URI
+            - JIGASI_HEALTH_CHECK_INTERVAL
+            - JIGASI_SIP_KEEP_ALIVE_METHOD
+            - JIGASI_ENABLE_SDES_SRTP
+            - ENABLE_TRANSCRIPTIONS
+            - JIGASI_TRANSCRIBER_ADVERTISE_URL
+            - JIGASI_TRANSCRIBER_RECORD_AUDIO
+            - JIGASI_TRANSCRIBER_SEND_TXT
+            - GC_PROJECT_ID
+            - GC_PRIVATE_KEY_ID
+            - GC_PRIVATE_KEY
+            - GC_CLIENT_EMAIL
+            - GC_CLIENT_ID
+            - GC_CLIENT_CERT_URL
+            - SENTRY_DSN="${JIGASI_SENTRY_DSN:-0}"
+            - SENTRY_ENVIRONMENT
+            - SENTRY_RELEASE
+            - TZ
+        depends_on:
+            - xmpp
+        networks:
+            meet.jitsi:
     jitsi-nginx:
         image: nginx:1.23
         depends_on:

diff --git a/frontend/.env b/frontend/.env
@@ -17,3 +17,4 @@ NEXT_PUBLIC_XMPP_MUC_DOMAIN=muc.meet.jitsi
 NEXT_PUBLIC_XMPP_AUTH_DOMAIN=auth.meet.jitsi
 NEXT_PUBLIC_JITSI_ROOM_PREFIX=
 NEXT_PUBLIC_ENVIRONMENT=dev
+NEXT_PUBLIC_TRANSCRIPTIONS_ENABLED=true
diff --git a/frontend/locales/ca/fishbowl.json b/frontend/locales/ca/fishbowl.json
@@ -205,5 +205,15 @@
             "during": "Durant",
             "duringTooltip": "Feedback rebut durant el fishbowl"
         }
+    },
+    "transcription" : {
+        "title": "Transcripció",
+        "disable": "Ocultar transcripció",
+        "enable": "Mostrar transcripció",
+        "loading": "Transcripció carregant...",
+        "modalTitle": "En quin idioma parles?",
+        "modalBody": "Abans de començar, necessitem que <span>indiquis en quin idioma participaràs</span> en aquesta conversa.",
+        "tooltip": "Si us plau, indica quin idioma parles.",
+        "translate": "Traduir"
     }
 }
diff --git a/frontend/locales/en/fishbowl.json b/frontend/locales/en/fishbowl.json
@@ -205,5 +205,15 @@
             "during": "During",
             "duringTooltip": "Feedback received during the fishbowl"
         }
+    },
+    "transcription" : {
+        "title": "Transcription",
+        "disable": "Disable transcription",
+        "enable": "Enable transcription",
+        "loading": "Transcription loading...",
+        "modalTitle": "What language do you speak?",
+        "modalBody": "Before we start, we need you to <span>indicate in which language you will be participating</span> in this conversation.",
+        "tooltip": "Please specify what language you speak.",
+        "translate": "Translate"
     }
 }
diff --git a/frontend/locales/es/fishbowl.json b/frontend/locales/es/fishbowl.json
@@ -205,5 +205,15 @@
             "during": "Durante",
             "duringTooltip": "Feedback recibido durante el fishbowl."
         }
+    },
+    "transcription" : {
+        "title": "Transcripción",
+        "disable": "Ocultar transcripción",
+        "enable": "Mostrar transcripción",
+        "loading": "Cargando transcripción...",
+        "modalTitle": "¿En qué idioma hablas?",
+        "modalBody": "Antes de comenzar, necesitamos que <span>indiques en qué idioma participarás</span> en esta conversación.",
+        "tooltip": "Por favor, indica qué idioma hablas.",
+        "translate": "Traducir"
     }
 }
diff --git a/frontend/locales/fr/fishbowl.json b/frontend/locales/fr/fishbowl.json
@@ -198,5 +198,15 @@
             "during": "During",
             "duringTooltip": "Feedback received during the fishbowl"
         }
+    },
+    "transcription" : {
+        "title" : "Transcription",
+        "disable": "Cacher la transcription",
+        "enable": "Afficher la transcription",
+        "loading": "Transcription du chargement...",
+        "modalTitle": "Dans quelle langue parlez-vous?",
+        "modalBody": "Avant de commencer, nous avons besoin que vous <span>indiquiez dans quelle langue vous participerez</span> à cette conversation.",
+        "tooltip": "Merci d'indiquer la langue que vous parlez.",
+        "translate": "Traduire"
     }
 }
diff --git a/frontend/src/components/App/ButtonMoreOptions/index.tsx b/frontend/src/components/App/ButtonMoreOptions/index.tsx
@@ -21,6 +21,7 @@ import SpeakerIcon from '@/ui/svg/speaker.svg';
 import VideoIcon from '@/ui/svg/video.svg';
 import CheckIcon from '@/ui/svg/checkmark.svg';
 import Cross from '@/ui/svg/cross.svg';
+import TranscriptionSVG from '@/ui/svg/transcription-icon.svg';
 import PermissionsAlert from '@/ui/svg/permissions-alert.svg';
 
 import {
@@ -36,6 +37,8 @@ import { useStooa } from '@/contexts/StooaManager';
 import { useModals } from '@/contexts/ModalsContext';
 import { useNavigatorType } from '@/hooks/useNavigatorType';
 import { supportsCaptureHandle } from '@/lib/helpers';
+import { useConference } from '@/jitsi/useConference';
+import { useUserAuth } from '@/user/auth/useUserAuth';
 
 interface Props {
   unlabeled?: boolean;
@@ -52,8 +55,13 @@ const ButtonMoreOptions: React.ForwardRefRenderFunction<ButtonHandle, Props> = (
   ref
 ) => {
   const [showDevices, setShowDevices] = useState(false);
-  const { setShowStopRecording, showStartRecording, setShowStartRecording, setShowFeedbackForm } =
-    useModals();
+  const {
+    setShowStopRecording,
+    setShowTranscriptionModal,
+    showStartRecording,
+    setShowStartRecording,
+    setShowFeedbackForm
+  } = useModals();
   const { deviceType } = useNavigatorType();
 
   const {
@@ -67,10 +75,36 @@ const ButtonMoreOptions: React.ForwardRefRenderFunction<ButtonHandle, Props> = (
     permissions
   } = useDevices();
 
-  const { isModerator, isRecording, feedbackAlert, gaveFeedback } = useStooa();
+  const {
+    isModerator,
+    isRecording,
+    feedbackAlert,
+    gaveFeedback,
+    isTranscriptionEnabled,
+    setIsTranscriptionEnabled,
+    setParticipantsActive,
+    isTranscriberJoined
+  } = useStooa();
+
+  const { stopTranscriptionEvent, startTranscriptionEvent, setConferenceTranscriptionLanguage } =
+    useConference();
+
+  const { getTranscriptionLanguageCookie } = useUserAuth();
 
   const { t } = useTranslation('fishbowl');
 
+  const getTranscriptionText = () => {
+    if (isTranscriptionEnabled && !isTranscriberJoined) {
+      return t('transcription.loading');
+    }
+
+    if (isTranscriptionEnabled) {
+      return t('transcription.disable');
+    } else {
+      return t('transcription.enable');
+    }
+  };
+
   const handleAudioInput = (event: React.MouseEvent) => {
     const { value } = event.target as HTMLButtonElement;
     selectAudioInputDevice(value);
@@ -106,6 +140,28 @@ const ButtonMoreOptions: React.ForwardRefRenderFunction<ButtonHandle, Props> = (
     setShowFeedbackForm(current => !current);
   };
 
+  const handleTranscriptionToggle = () => {
+    const transcriptionCookie = getTranscriptionLanguageCookie();
+
+    if (!transcriptionCookie) {
+      setShowTranscriptionModal(true);
+      return;
+    }
+
+    if (isTranscriptionEnabled) {
+      stopTranscriptionEvent();
+      setIsTranscriptionEnabled(false);
+    } else {
+      startTranscriptionEvent();
+      setConferenceTranscriptionLanguage(transcriptionCookie);
+      if (deviceType === 'Desktop') {
+        setParticipantsActive(true);
+      }
+      setIsTranscriptionEnabled(true);
+      setShowTranscriptionModal(false);
+    }
+  };
+
   // Used imperativeHandle to make the parents able to call the handleShowDevices
   // this way we keep the state inside the button config component
   useImperativeHandle(ref, () => ({
@@ -160,6 +216,19 @@ const ButtonMoreOptions: React.ForwardRefRenderFunction<ButtonHandle, Props> = (
                   {t('feedback.title')}
                 </button>
               )}
+
+              {!prejoin && process.env.NEXT_PUBLIC_TRANSCRIPTIONS_ENABLED === 'true' && (
+                <button
+                  disabled={gaveFeedback}
+                  data-testid="transcription-button"
+                  className="sticky-button sticky-button--transcription"
+                  onClick={() => handleTranscriptionToggle()}
+                >
+                  <TranscriptionSVG />
+                  {getTranscriptionText()}
+                </button>
+              )}
+
               {isModerator && supportsCaptureHandle() && deviceType === 'Desktop' && !prejoin && (
                 <button
                   data-testid="recording-button"

diff --git a/frontend/src/components/App/ButtonMoreOptions/styles.ts b/frontend/src/components/App/ButtonMoreOptions/styles.ts
@@ -111,12 +111,13 @@ const Selector = styled.div`
     background-color: ${COLOR_NEUTRO_100};
     transition: background-color 0.2s ease-in-out;
 
+    border-bottom: 1px solid ${COLOR_NEUTRO_600};
+
     & .sticky-button {
       width: 100%;
       padding: ${space()} ${space(2)};
       display: flex;
       align-items: center;
-      border-bottom: 1px solid ${COLOR_NEUTRO_600};
 
       text-align: left;