feat: Initial implementation of Gladia transcriptions to BBB 2.7

akka-bbb-apps/src/main/scala/org/bigbluebutton/core/apps/pads/PadUpdatePubMsgHdlr.scala

@@ -21,7 +21,7 @@ trait PadUpdatePubMsgHdlr {
       bus.outGW.send(msgEvent)
     }
 
-    if (Pads.hasAccess(liveMeeting, msg.body.externalId, msg.header.userId)) {
+    if (Pads.hasAccess(liveMeeting, msg.body.externalId, msg.header.userId) || msg.body.transcript == true) {
       Pads.getGroup(liveMeeting.pads, msg.body.externalId) match {
         case Some(group) => broadcastEvent(group.groupId, msg.body.externalId, msg.body.text)
         case _           =>

akka-bbb-apps/src/main/scala/org/bigbluebutton/core/apps/users/SetUserSpeechOptionsMsgHdlr.scala

@@ -0,0 +1,41 @@
+package org.bigbluebutton.core.apps.users
+
+import org.bigbluebutton.common2.msgs._
+import org.bigbluebutton.core.models.{ UserState, Users2x }
+import org.bigbluebutton.core.running.{ LiveMeeting, OutMsgRouter }
+import org.bigbluebutton.core.apps.{ PermissionCheck, RightsManagementTrait }
+import org.bigbluebutton.core.domain.MeetingState2x
+
+trait SetUserSpeechOptionsMsgHdlr extends RightsManagementTrait {
+  this: UsersApp =>
+
+  val liveMeeting: LiveMeeting
+  val outGW: OutMsgRouter
+
+  def handleSetUserSpeechOptionsReqMsg(msg: SetUserSpeechOptionsReqMsg): Unit = {
+    log.info("handleSetUserSpeechOptionsReqMsg: partialUtterances={} minUtteranceLength={} userId={}", msg.body.partialUtterances, msg.body.minUtteranceLength, msg.header.userId)
+
+    def broadcastUserSpeechOptionsChanged(user: UserState, partialUtterances: Boolean, minUtteranceLength: Int): Unit = {
+      val routingChange = Routing.addMsgToClientRouting(
+        MessageTypes.BROADCAST_TO_MEETING,
+        liveMeeting.props.meetingProp.intId, user.intId
+      )
+      val envelopeChange = BbbCoreEnvelope(UserSpeechOptionsChangedEvtMsg.NAME, routingChange)
+      val headerChange = BbbClientMsgHeader(UserSpeechOptionsChangedEvtMsg.NAME, liveMeeting.props.meetingProp.intId, user.intId)
+
+      val bodyChange = UserSpeechOptionsChangedEvtMsgBody(partialUtterances, minUtteranceLength)
+      val eventChange = UserSpeechOptionsChangedEvtMsg(headerChange, bodyChange)
+      val msgEventChange = BbbCommonEnvCoreMsg(envelopeChange, eventChange)
+      outGW.send(msgEventChange)
+    }
+
+    for {
+      user <- Users2x.findWithIntId(liveMeeting.users2x, msg.header.userId)
+    } yield {
+      var changeLocale: Option[UserState] = None;
+      //changeLocale = Users2x.setUserSpeechLocale(liveMeeting.users2x, msg.header.userId, msg.body.locale)
+      broadcastUserSpeechOptionsChanged(user, msg.body.partialUtterances, msg.body.minUtteranceLength)
+    }
+
+  }
+}

akka-bbb-apps/src/main/scala/org/bigbluebutton/core/apps/users/UsersApp.scala

@@ -150,6 +150,7 @@ class UsersApp(
   with RegisterUserReqMsgHdlr
   with ChangeUserRoleCmdMsgHdlr
   with SetUserSpeechLocaleMsgHdlr
+  with SetUserSpeechOptionsMsgHdlr
   with SyncGetUsersMeetingRespMsgHdlr
   with LogoutAndEndMeetingCmdMsgHdlr
   with SetRecordingStatusCmdMsgHdlr

akka-bbb-apps/src/main/scala/org/bigbluebutton/core/models/AudioCaptions.scala

@@ -7,12 +7,10 @@ import org.bigbluebutton.SystemConfiguration
 object AudioCaptions extends SystemConfiguration {
   def setFloor(audioCaptions: AudioCaptions, userId: String) = audioCaptions.floor = userId
 
-  def isFloor(audioCaptions: AudioCaptions, userId: String) = audioCaptions.floor == userId
+  def isFloor(audioCaptions: AudioCaptions, userId: String) = true
 
   def parseTranscript(transcript: String): String = {
-    val words = transcript.split("\\s+") // Split on whitespaces
-    val lines = words.grouped(transcriptWords).toArray // Group each X words into lines
-    lines.takeRight(transcriptLines).map(l => l.mkString(" ")).mkString("\n") // Join the last X lines
+    transcript
   }
 
   /*

akka-bbb-apps/src/main/scala/org/bigbluebutton/core/pubsub/senders/ReceivedJsonMsgHandlerActor.scala

@@ -113,6 +113,8 @@ class ReceivedJsonMsgHandlerActor(
         routeGenericMsg[ChangeUserMobileFlagReqMsg](envelope, jsonNode)
       case SetUserSpeechLocaleReqMsg.NAME =>
         routeGenericMsg[SetUserSpeechLocaleReqMsg](envelope, jsonNode)
+      case SetUserSpeechOptionsReqMsg.NAME =>
+        routeGenericMsg[SetUserSpeechOptionsReqMsg](envelope, jsonNode)
       case SelectRandomViewerReqMsg.NAME =>
         routeGenericMsg[SelectRandomViewerReqMsg](envelope, jsonNode)
 

akka-bbb-apps/src/main/scala/org/bigbluebutton/core/running/MeetingActor.scala

@@ -399,6 +399,7 @@ class MeetingActor(
       case m: ChangeUserPinStateReqMsg      => usersApp.handleChangeUserPinStateReqMsg(m)
       case m: ChangeUserMobileFlagReqMsg    => usersApp.handleChangeUserMobileFlagReqMsg(m)
       case m: SetUserSpeechLocaleReqMsg     => usersApp.handleSetUserSpeechLocaleReqMsg(m)
+      case m: SetUserSpeechOptionsReqMsg => usersApp.handleSetUserSpeechOptionsReqMsg(m)
 
       // Client requested to eject user
       case m: EjectUserFromMeetingCmdMsg =>

bbb-common-message/src/main/scala/org/bigbluebutton/common2/msgs/PadsMsgs.scala

@@ -107,7 +107,7 @@ case class PadTailEvtMsgBody(externalId: String, tail: String)
 // client -> apps
 object PadUpdatePubMsg { val NAME = "PadUpdatePubMsg" }
 case class PadUpdatePubMsg(header: BbbClientMsgHeader, body: PadUpdatePubMsgBody) extends StandardMsg
-case class PadUpdatePubMsgBody(externalId: String, text: String)
+case class PadUpdatePubMsgBody(externalId: String, text: String, transcript: Boolean)
 
 // apps -> pads
 object PadUpdateCmdMsg { val NAME = "PadUpdateCmdMsg" }

bbb-common-message/src/main/scala/org/bigbluebutton/common2/msgs/UsersMsgs.scala

@@ -531,3 +531,11 @@ case class SetUserSpeechLocaleReqMsgBody(locale: String, provider: String)
 object UserSpeechLocaleChangedEvtMsg { val NAME = "UserSpeechLocaleChangedEvtMsg" }
 case class UserSpeechLocaleChangedEvtMsg(header: BbbClientMsgHeader, body: UserSpeechLocaleChangedEvtMsgBody) extends BbbCoreMsg
 case class UserSpeechLocaleChangedEvtMsgBody(locale: String, provider: String)
+
+object SetUserSpeechOptionsReqMsg { val NAME = "SetUserSpeechOptionsReqMsg" }
+case class SetUserSpeechOptionsReqMsg(header: BbbClientMsgHeader, body: SetUserSpeechOptionsReqMsgBody) extends StandardMsg
+case class SetUserSpeechOptionsReqMsgBody(partialUtterances: Boolean, minUtteranceLength: Int)
+
+object UserSpeechOptionsChangedEvtMsg { val NAME = "UserSpeechOptionsChangedEvtMsg" }
+case class UserSpeechOptionsChangedEvtMsg(header: BbbClientMsgHeader, body: UserSpeechOptionsChangedEvtMsgBody) extends BbbCoreMsg
+case class UserSpeechOptionsChangedEvtMsgBody(partialUtterances: Boolean, minUtteranceLength: Int)

bbb-transcription-controller.placeholder.sh

@@ -1 +1 @@
-git clone --branch v0.1.0 --depth 1 https://github.com/bigbluebutton/bbb-transcription-controller bbb-transcription-controller
+git clone --branch v0.2.1 --depth 1 https://github.com/bigbluebutton/bbb-transcription-controller bbb-transcription-controller

bigbluebutton-config/bin/bbb-conf

@@ -1729,7 +1729,7 @@ if [ -n "$HOST" ]; then
     sudo yq w -i /usr/local/bigbluebutton/bbb-webrtc-sfu/config/default.yml freeswitch.esl_password "$ESL_PASSWORD"
     sudo xmlstarlet edit --inplace --update 'configuration/settings//param[@name="password"]/@value' --value $ESL_PASSWORD /opt/freeswitch/etc/freeswitch/autoload_configs/event_socket.conf.xml
     if [ -f /usr/local/bigbluebutton/bbb-transcription-controller/config/default.yml ]; then
-        sudo yq w -i /usr/local/bigbluebutton/bbb-transcription-controller/config/default.yml freeswitch.esl_password "$ESL_PASSWORD"
+        sudo yq w -i /usr/local/bigbluebutton/bbb-transcription-controller/config/default.yml freeswitch.password "$ESL_PASSWORD"
     fi
 
     echo "Restarting BigBlueButton $BIGBLUEBUTTON_RELEASE ..."

bigbluebutton-html5/imports/api/audio-captions/server/handlers/transcriptUpdated.js

@@ -1,12 +1,38 @@
 import setTranscript from '/imports/api/audio-captions/server/modifiers/setTranscript';
+import updateTranscriptPad from '/imports/api/pads/server/methods/updateTranscriptPad';
+import Users from '/imports/api/users';
+
+const TRANSCRIPTION_DEFAULT_PAD = Meteor.settings.public.captions.defaultPad;
+
+const formatDate = (dStr) => {
+  return ("00" + dStr).substr(-2,2);
+};
 
 export default async function transcriptUpdated({ header, body }) {
-  const { meetingId } = header;
+  const {
+    meetingId,
+    userId,
+  } = header;
 
   const {
     transcriptId,
     transcript,
+    locale,
+    result,
   } = body;
 
-  await setTranscript(meetingId, transcriptId, transcript);
+  if (result) {
+    const user = Users.findOne({ userId }, { fields: { name: 1 } });
+    const userName = user?.name || '??';
+
+    const dt = new Date(Date.now());
+    const hours = formatDate(dt.getHours()),
+          minutes = formatDate(dt.getMinutes()),
+          seconds = formatDate(dt.getSeconds());
+
+    const userSpoke = `\n ${userName} (${hours}:${minutes}:${seconds}): ${transcript}`;
+    updateTranscriptPad(meetingId, userId, TRANSCRIPTION_DEFAULT_PAD, userSpoke);
+  }
+
+  await setTranscript(userId, meetingId, transcriptId, transcript, locale);
 }

bigbluebutton-html5/imports/api/audio-captions/server/modifiers/setTranscript.js

@@ -1,28 +1,30 @@
 import { check } from 'meteor/check';
 import AudioCaptions from '/imports/api/audio-captions';
+import Users from '/imports/api/users';
 import Logger from '/imports/startup/server/logger';
 
-export default async function setTranscript(meetingId, transcriptId, transcript) {
+export default async function setTranscript(userId, meetingId, transcriptId, transcript, locale) {
   try {
     check(meetingId, String);
     check(transcriptId, String);
     check(transcript, String);
 
-    const selector = { meetingId };
+    const selector = { meetingId, transcriptId };
 
     const modifier = {
       $set: {
-        transcriptId,
         transcript,
+        lastUpdated: Math.floor(new Date().getTime()/1000),
+        locale,
       },
     };
 
     const numberAffected = await AudioCaptions.upsertAsync(selector, modifier);
 
     if (numberAffected) {
-      Logger.debug(`Set transcriptId=${transcriptId} transcript=${transcript} meeting=${meetingId}`);
+      Logger.debug(`Set transcriptId=${transcriptId} transcript=${transcript} meeting=${meetingId} locale=${locale}`);
     } else {
-      Logger.debug(`Upserted transcriptId=${transcriptId} transcript=${transcript} meeting=${meetingId}`);
+      Logger.debug(`Upserted transcriptId=${transcriptId} transcript=${transcript} meeting=${meetingId} locale=${locale}`);
     }
   } catch (err) {
     Logger.error(`Setting audio captions transcript to the collection: ${err}`);

bigbluebutton-html5/imports/api/pads/server/methods/updatePad.js

@@ -17,6 +17,7 @@ export default function updatePad(meetingId, userId, externalId, text) {
     const payload = {
       externalId,
       text,
+      transcript: false,
     };
 
     RedisPubSub.publishUserMessage(CHANNEL, EVENT_NAME, meetingId, userId, payload);

bigbluebutton-html5/imports/api/pads/server/methods/updateTranscriptPad.js

@@ -0,0 +1,29 @@
+import RedisPubSub from '/imports/startup/server/redis';
+import { Meteor } from 'meteor/meteor';
+import { check } from 'meteor/check';
+import Logger from '/imports/startup/server/logger';
+
+export default function updateTranscriptPad(meetingId, userId, externalId, text) {
+  const REDIS_CONFIG = Meteor.settings.private.redis;
+  const CHANNEL = REDIS_CONFIG.channels.toAkkaApps;
+  const EVENT_NAME = 'PadUpdatePubMsg';
+
+  try {
+    check(meetingId, String);
+    check(userId, String);
+    check(externalId, String);
+    check(text, String);
+
+    // Send a special boolean denoting this was updated by the transcript system
+    // this way we can write it in the 'presenter' pad and still block manual updates by viewers
+    const payload = {
+      externalId,
+      text,
+      transcript: true,
+    };
+
+    RedisPubSub.publishUserMessage(CHANNEL, EVENT_NAME, meetingId, userId, payload);
+  } catch (err) {
+    Logger.error(`Exception while invoking method updateTranscriptPad ${err.stack}`);
+  }
+}

bigbluebutton-html5/imports/api/users-settings/server/methods/addUserSettings.js

@@ -65,6 +65,10 @@ const currentParameters = [
   'bbb_hide_nav_bar',
   'bbb_change_layout',
   'bbb_direct_leave_button',
+  // TRANSCRIPTION
+  'bbb_transcription_partial_utterances',
+  'bbb_transcription_min_utterance_length',
+  'bbb_transcription_provider',
 ];
 
 function valueParser(val) {

bigbluebutton-html5/imports/api/users/server/methods.js

@@ -1,6 +1,7 @@
 import { Meteor } from 'meteor/meteor';
 import validateAuthToken from './methods/validateAuthToken';
 import setSpeechLocale from './methods/setSpeechLocale';
+import setSpeechOptions from './methods/setSpeechOptions';
 import setMobileUser from './methods/setMobileUser';
 import setEmojiStatus from './methods/setEmojiStatus';
 import changeAway from './methods/changeAway';
@@ -19,6 +20,7 @@ import clearAllUsersEmoji from './methods/clearAllUsersEmoji';
 
 Meteor.methods({
   setSpeechLocale,
+  setSpeechOptions,
   setMobileUser,
   setEmojiStatus,
   clearAllUsersEmoji,

bigbluebutton-html5/imports/api/users/server/methods/setSpeechLocale.js

@@ -23,7 +23,7 @@ export default function setSpeechLocale(locale, provider) {
       provider: provider !== 'webspeech' ? provider : '',
     };
 
-    if (LANGUAGES.includes(locale) || locale === '') {
+    if (LANGUAGES.includes(locale) || locale === '' || locale === 'auto') {
       RedisPubSub.publishUserMessage(CHANNEL, EVENT_NAME, meetingId, requesterUserId, payload);
     }
   } catch (err) {

bigbluebutton-html5/imports/api/users/server/methods/setSpeechOptions.js

@@ -0,0 +1,30 @@
+import { check } from 'meteor/check';
+import Logger from '/imports/startup/server/logger';
+import RedisPubSub from '/imports/startup/server/redis';
+import { extractCredentials } from '/imports/api/common/server/helpers';
+
+export default async function setSpeechOptions(partialUtterances, minUtteranceLength) {
+  try {
+    const { meetingId, requesterUserId } = extractCredentials(this.userId);
+
+    const REDIS_CONFIG = Meteor.settings.private.redis;
+    const CHANNEL = REDIS_CONFIG.channels.toAkkaApps;
+    const EVENT_NAME = 'SetUserSpeechOptionsReqMsg';
+
+    Logger.info(`Setting speech options for ${meetingId} ${requesterUserId} ${partialUtterances} ${minUtteranceLength}`);
+
+    check(meetingId, String);
+    check(requesterUserId, String);
+    check(partialUtterances, Boolean);
+    check(minUtteranceLength, Number);
+
+    const payload = {
+      partialUtterances,
+      minUtteranceLength,
+    };
+
+    RedisPubSub.publishUserMessage(CHANNEL, EVENT_NAME, meetingId, requesterUserId, payload);
+  } catch (e) {
+    Logger.error(e);
+  }
+}

bigbluebutton-html5/imports/startup/client/base.jsx

@@ -26,6 +26,7 @@ import BBBStorage from '/imports/ui/services/storage';
 const CHAT_CONFIG = Meteor.settings.public.chat;
 const PUBLIC_CHAT_ID = CHAT_CONFIG.public_id;
 const USER_WAS_EJECTED = 'userWasEjected';
+const CAPTIONS_ALWAYS_VISIBLE = Meteor.settings.public.app.audioCaptions.alwaysVisible;
 
 const HTML = document.getElementsByTagName('html')[0];
 
@@ -98,6 +99,7 @@ class Base extends Component {
     fullscreenChangedEvents.forEach((event) => {
       document.addEventListener(event, this.handleFullscreenChange);
     });
+    Session.set('audioCaptions', CAPTIONS_ALWAYS_VISIBLE);
     Session.set('isFullscreen', false);
   }
 

bigbluebutton-html5/imports/ui/components/app/component.jsx

@@ -52,6 +52,7 @@ import NotesContainer from '/imports/ui/components/notes/container';
 import DEFAULT_VALUES from '../layout/defaultValues';
 import AppService from '/imports/ui/components/app/service';
 import TimerService from '/imports/ui/components/timer/service';
+import SpeechService from '/imports/ui/components/audio/captions/speech/service';
 
 const MOBILE_MEDIA = 'only screen and (max-width: 40em)';
 const APP_CONFIG = Meteor.settings.public.app;
@@ -171,6 +172,7 @@ class App extends Component {
       intl,
       layoutContextDispatch,
       isRTL,
+      transcriptionSettings,
     } = this.props;
     const { browserName } = browserInfo;
     const { osName } = deviceInfo;
@@ -232,6 +234,18 @@ class App extends Component {
         TimerService.OFFSET_INTERVAL);
     }
 
+    if (transcriptionSettings) {
+      const { partialUtterances, minUtteranceLength } = transcriptionSettings;
+      if (partialUtterances !== undefined || minUtteranceLength !== undefined) {
+        logger.info({ logCode: 'app_component_set_speech_options' }, 'Setting initial speech options');
+
+        Settings.transcription.partialUtterances = partialUtterances ? true : false;
+        Settings.transcription.minUtteranceLength = parseInt(minUtteranceLength, 10);
+
+        SpeechService.setSpeechOptions(Settings.transcription.partialUtterances, Settings.transcription.minUtteranceLength);
+      }
+    }
+
     logger.info({ logCode: 'app_component_componentdidmount' }, 'Client loaded successfully');
   }
 

bigbluebutton-html5/imports/ui/components/app/container.jsx

@@ -3,7 +3,7 @@ import { withTracker } from 'meteor/react-meteor-data';
 import Auth from '/imports/ui/services/auth';
 import Users from '/imports/api/users';
 import Meetings, { LayoutMeetings } from '/imports/api/meetings';
-import AudioCaptionsLiveContainer from '/imports/ui/components/audio/captions/live/container';
+import AudioCaptionsLiveContainer from '/imports/ui/components/audio/captions/history/container';
 import AudioCaptionsService from '/imports/ui/components/audio/captions/service';
 import { notify } from '/imports/ui/services/notification';
 import CaptionsContainer from '/imports/ui/components/captions/live/container';
@@ -281,6 +281,11 @@ export default withTracker(() => {
 
   const isPresenter = currentUser?.presenter;
 
+  const transcriptionSettings = {
+    partialUtterances: getFromUserSettings('bbb_transcription_partial_utterances'),
+    minUtteranceLength: getFromUserSettings('bbb_transcription_min_utterance_length'),
+  };
+
   return {
     captions: CaptionsService.isCaptionsActive() ? <CaptionsContainer /> : null,
     audioCaptions: AudioCaptionsService.getAudioCaptions() ? <AudioCaptionsLiveContainer /> : null,
@@ -325,5 +330,6 @@ export default withTracker(() => {
     hideActionsBar: getFromUserSettings('bbb_hide_actions_bar', false),
     ignorePollNotifications: Session.get('ignorePollNotifications'),
     isSharedNotesPinned: MediaService.shouldShowSharedNotes(),
+    transcriptionSettings,
   };
 })(AppContainer);