Sends json-messages for translations & Sets the participant translati…

…on language on presence update (#124)

* Sends json transcript messages in json extension

(cherry picked from commit aca02e1)

* Rename constant name, log fixes

* Removes createEncapsulatingObject, rename variables

* Publishes hardcoded translations to the chatroom

* Publishes translation results as json-message and minor fixes

* Sets the participant translation language on presence update of member

* enable translation in config, removes language on removal from presence

* Updates pom.xml and minor fix for translation config

* Variable synchronizations and code style fixes

jigasi-home/sip-communicator.properties

@@ -115,6 +115,9 @@ org.jitsi.jigasi.xmpp.acc.VIDEO_CALLING_DISABLED=true
 # org.jitsi.jigasi.transcription.SEND_JSON=true
 # org.jitsi.jigasi.transcription.SEND_TXT=false
 
+# translation
+# org.jitsi.jigasi.transcription.ENABLE_TRANSLATION=false
+
 # record audio. Currently only wav format is supported
 # org.jitsi.jigasi.transcription.RECORD_AUDIO=false
 # org.jitsi.jigasi.transcription.RECORD_AUDIO_FORMAT=wav

pom.xml

@@ -20,7 +20,7 @@
   <properties>
     <assembly.skipAssembly>true</assembly.skipAssembly>
     <smack.version>4.2.1</smack.version>
-    <jitsi-desktop.version>2.13.97cbdf3</jitsi-desktop.version>
+    <jitsi-desktop.version>2.13.67d8c8b</jitsi-desktop.version>
   </properties>
 
   <dependencies>
@@ -34,6 +34,11 @@
       <artifactId>google-cloud-speech</artifactId>
       <version>0.49.0-alpha</version>
     </dependency>
+    <dependency>
+      <groupId>com.google.cloud</groupId>
+      <artifactId>google-cloud-translate</artifactId>
+      <version>1.31.0</version>
+    </dependency>
     <dependency>
       <groupId>dom4j</groupId>
       <artifactId>dom4j</artifactId>

src/main/java/org/jitsi/jigasi/AbstractGatewaySession.java

@@ -21,6 +21,7 @@
 import net.java.sip.communicator.service.protocol.*;
 import net.java.sip.communicator.service.protocol.event.*;
 import net.java.sip.communicator.util.*;
+import org.jivesoftware.smack.packet.*;
 
 import java.util.*;
 
@@ -292,6 +293,18 @@ void notifyChatRoomMemberLeft(ChatRoomMember member)
          */
     }
 
+    /**
+     * Method called by {@link JvbConference} to notify session that a member
+     * has sent an updated presence packet.
+     *
+     * @param member the member who left the JVB conference
+     * @param presence the updated presence of the member
+     */
+    void notifyChatRoomMemberUpdated(ChatRoomMember member, Presence presence)
+    {
+        // We don't need to do anything here.
+    }
+
     /**
      * Returns the cumulative number of participants that were active during
      * this session including the focus.

src/main/java/org/jitsi/jigasi/JvbConference.java

@@ -801,6 +801,8 @@ else if(ChatRoomMemberPresenceChangeEvent.MEMBER_UPDATED
                     Presence presence
                         = ((ChatRoomMemberJabberImpl) member).getLastPresence();
 
+                    gatewaySession.notifyChatRoomMemberUpdated(member, presence);
+
                     RecordingStatus rs = presence.getExtension(
                         RecordingStatus.ELEMENT_NAME,
                         RecordingStatus.NAMESPACE);

src/main/java/org/jitsi/jigasi/TranscriptionGatewaySession.java

@@ -18,6 +18,7 @@
 package org.jitsi.jigasi;
 
 import net.java.sip.communicator.impl.protocol.jabber.*;
+import net.java.sip.communicator.impl.protocol.jabber.extensions.jitsimeet.*;
 import net.java.sip.communicator.service.protocol.*;
 import net.java.sip.communicator.service.protocol.event.*;
 import net.java.sip.communicator.service.protocol.media.*;
@@ -26,6 +27,7 @@
 import org.jitsi.service.neomedia.*;
 import org.jitsi.service.neomedia.device.*;
 import org.jitsi.util.*;
+import org.jivesoftware.smack.packet.Presence;
 import org.jxmpp.jid.*;
 import org.jxmpp.jid.impl.*;
 import org.jxmpp.stringprep.*;
@@ -41,7 +43,8 @@
 public class TranscriptionGatewaySession
     extends AbstractGatewaySession
     implements TranscriptionListener,
-               TranscriptionEventListener
+               TranscriptionEventListener,
+               TranslationResultListener
 {
     /**
      * The logger of this class
@@ -120,6 +123,7 @@ void onConferenceCallInvited(Call incomingCall)
         // We got invited to a room, we can tell the transcriber
         // the room name, url and start listening
         transcriber.addTranscriptionListener(this);
+        transcriber.addTranslationListener(this);
         transcriber.setRoomName(getJvbRoomName());
         transcriber.setRoomUrl(getMeetingUrl());
 
@@ -257,6 +261,30 @@ void notifyChatRoomMemberLeft(ChatRoomMember chatMember)
         this.transcriber.participantLeft(identifier);
     }
 
+    @Override
+    void notifyChatRoomMemberUpdated(ChatRoomMember chatMember, Presence presence)
+    {
+        super.notifyChatRoomMemberUpdated(chatMember, presence);
+
+        String identifier = getParticipantIdentifier(chatMember);
+        TranslationLanguageExtension translationLanguageExtension
+            = presence.getExtension(
+                TranslationLanguageExtension.ELEMENT_NAME,
+                TranslationLanguageExtension.NAMESPACE);
+
+        if(translationLanguageExtension != null)
+        {
+            String language
+                = translationLanguageExtension.getTranslationLanguage();
+
+            this.transcriber.updateParticipantLanguage(identifier, language);
+        }
+        else
+        {
+            this.transcriber.updateParticipantLanguage(identifier, null);
+        }
+    }
+
     @Override
     void notifyConferenceMemberJoined(ConferenceMember conferenceMember)
     {
@@ -301,6 +329,18 @@ public void notify(TranscriptionResult result)
         sendTranscriptionResultToRoom(result);
     }
 
+    /**
+     * This method will be called by the {@link TranslationManager} every time a
+     * final transcription result is translated.
+     *
+     * @param result the translation result which has come in
+     */
+    @Override
+    public void notify(TranslationResult result)
+    {
+        sendTranslationResultToRoom(result);
+    }
+
     /**
      * This method will be called by the {@link Transcriber} when it is done
      * and thus no more results will come in
@@ -531,6 +571,16 @@ private void sendTranscriptionResultToRoom(TranscriptionResult result)
         handler.publishTranscriptionResult(this.chatRoom, result);
     }
 
+    /**
+     * Send a {@link TranslationResult} to the {@link ChatRoom}
+     *
+     * @param result the {@link TranscriptionResult} to send
+     */
+    private void sendTranslationResultToRoom(TranslationResult result)
+    {
+        handler.publishTranslationResult(this.chatRoom, result);
+    }
+
     @Override
     public String getMucDisplayName()
     {

src/main/java/org/jitsi/jigasi/transcription/AbstractTranscriptPublisher.java

@@ -205,8 +205,8 @@ protected void sendMessage(ChatRoom chatRoom, T message)
         try
         {
             chatRoom.sendMessage(chatRoomMessage);
-            if (logger.isDebugEnabled())
-                logger.debug("Sending message: \"" + messageString + "\"");
+            if (logger.isTraceEnabled())
+                logger.trace("Sending message: \"" + messageString + "\"");
         }
         catch (OperationFailedException e)
         {

src/main/java/org/jitsi/jigasi/transcription/GoogleCloudTranslationService.java

@@ -0,0 +1,65 @@
+/*
+ * Jigasi, the JItsi GAteway to SIP.
+ *
+ * Copyright @ 2018 Atlassian Pty Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.jitsi.jigasi.transcription;
+
+import com.google.cloud.translate.*;
+import com.google.cloud.translate.Translate.*;
+
+/**
+ * Implements a {@link TranslationService} which will use Google Cloud
+ * translate API to translate the given text from one language to another.
+ *
+ * Requires google cloud sdk on the machine running the JVM running
+ * this application. See
+ * https://cloud.google.com/translate/docs/quickstart
+ * for details
+ *
+ * @author Praveen Kumar Gupta
+ */
+public class GoogleCloudTranslationService
+    implements TranslationService
+{
+
+    /**
+    * The translation model to be used for translation.
+    * Possible values are {@code base} and {@code nmt}
+    */
+    private static final TranslateOption model = TranslateOption.model("nmt");
+
+    /**
+    * Translation service object for accessing the cloud translate API.
+    */
+    private static final Translate translator
+        = TranslateOptions.getDefaultInstance().getService();
+
+    /**
+     * {@inheritDoc}
+     */
+    @Override
+    public String translate(String sourceText,
+                            String sourceLang, String targetLang)
+    {
+        TranslateOption srcLang = TranslateOption.sourceLanguage(sourceLang);
+        TranslateOption tgtLang = TranslateOption.targetLanguage(targetLang);
+
+        Translation translation = translator.translate(
+            sourceText, srcLang, tgtLang, model);
+
+        return translation.getTranslatedText();
+    }
+}

src/main/java/org/jitsi/jigasi/transcription/LocalJsonTranscriptHandler.java

@@ -205,9 +205,18 @@ public class LocalJsonTranscriptHandler
     public final static String JSON_KEY_TYPE = "type";
 
     /**
-     * This field stores the value of the type of the muc message as a string
+     * This field stores the value of the type of the muc message  for
+     * a transcription result to be sent.
      */
-    public final static String JSON_VALUE_TYPE = "transcription-result";
+    public final static String JSON_VALUE_TYPE_TRANSCRIPTION_RESULT
+        = "transcription-result";
+
+    /**
+     * This field stores the value of the type of muc message for
+     * a translation result to be sent.
+     */
+    public final static String JSON_VALUE_TYPE_TRANSLATION_RESULT
+        = "translation-result";
 
     @Override
     public JSONFormatter getFormatter()
@@ -218,7 +227,15 @@ public JSONFormatter getFormatter()
     @Override
     public void publish(ChatRoom room, TranscriptionResult result)
     {
-        JSONObject eventObject = createJSONObject(result);
+        JSONObject eventObject = createTranscriptionJSONObject(result);
+
+        super.sendJsonMessage(room, eventObject);
+    }
+
+    @Override
+    public void publish(ChatRoom room, TranslationResult result)
+    {
+        JSONObject eventObject = createTranslationJSONObject(result);
 
         super.sendJsonMessage(room, eventObject);
     }
@@ -229,19 +246,44 @@ public void publish(ChatRoom room, TranscriptionResult result)
      * @return json object representing the <tt>TranscriptionResult</>.
      */
     @SuppressWarnings("unchecked")
-    public static JSONObject createJSONObject(TranscriptionResult result)
+    public static JSONObject createTranscriptionJSONObject(
+        TranscriptionResult result)
     {
         JSONObject eventObject = new JSONObject();
         SpeechEvent event = new SpeechEvent(Instant.now(), result);
 
         addEventDescriptions(eventObject, event);
         addAlternatives(eventObject, event);
 
-        eventObject.put(JSON_KEY_TYPE, JSON_VALUE_TYPE);
+        eventObject.put(JSON_KEY_TYPE, JSON_VALUE_TYPE_TRANSCRIPTION_RESULT);
 
         return eventObject;
     }
 
+    /**
+     *Creates a json object representing the <tt>TranslationResult</tt>.
+     *
+     * @param result the object to be used to produce json.
+     * @return json object representing the <tt>TranslationResult</tt>.
+     */
+    @SuppressWarnings("unchecked")
+    private static JSONObject createTranslationJSONObject(
+        TranslationResult result)
+    {
+        JSONObject eventObject = new JSONObject();
+        SpeechEvent event = new SpeechEvent(Instant.now(),
+            result.getTranscriptionResult());
+
+        addEventDescriptions(eventObject, event);
+
+        eventObject.put(JSON_KEY_TYPE, JSON_VALUE_TYPE_TRANSLATION_RESULT);
+        eventObject.put(JSON_KEY_EVENT_LANGUAGE, result.getLanguage());
+        eventObject.put(JSON_KEY_ALTERNATIVE_TEXT, result.getTranslatedText());
+        eventObject.put(JSON_KEY_EVENT_MESSAGE_ID,
+                result.getTranscriptionResult().getMessageID().toString());
+
+        return eventObject;
+    }
     @Override
     public Promise getPublishPromise()
     {

src/main/java/org/jitsi/jigasi/transcription/LocalTxtTranscriptHandler.java

@@ -195,6 +195,21 @@ public void publish(ChatRoom chatRoom, TranscriptionResult result)
         super.sendMessage(chatRoom, toSend);
     }
 
+    /**
+     * Translated Speech-to-text results can be published to the Chatroom
+     * as plain-text here.
+     *
+     * {@inheritDoc}
+     */
+    @Override
+    public void publish(ChatRoom chatRoom, TranslationResult result)
+    {
+        /*
+         * We do not send the translated speech-to-text results to the Chatroom
+         * as it will flood the Chatroom with messages.
+         */
+    }
+
     @Override
     public Promise getPublishPromise()
     {