[Vision Glass] Voice button can be used for search and text input

Set up the connection to the Huawei Speech Recognition service. The voice button on the phone UI will perform a search if the user is not currently writing on a text field (and therefore the keyboard is not visible). When the keyboard is visible, the voice button will capture text which will be entered into the text field that the user is editing.
Igalia · Jun 14, 2024 · a036b74 · a036b74
1 parent 8bf0314
commit a036b74
Show file tree

Hide file tree

Showing 4 changed files with 48 additions and 25 deletions.
diff --git a/app/build.gradle b/app/build.gradle
@@ -9,8 +9,9 @@ apply from: "$project.rootDir/tools/gradle/versionCode.gradle"
 apply plugin: 'kotlin-android'
 apply plugin: "org.mozilla.telemetry.glean-gradle-plugin"
 
-// Apply AGConnect plugin only for Hvr builds
-if (getGradle().getStartParameter().getTaskRequests().toString() =~ /(h|H)vr/) {
+// Apply AGConnect plugin only for Huawei builds
+if (getGradle().getStartParameter().getTaskRequests().toString() =~ /[Hh]vr/
+        || getGradle().getStartParameter().getTaskRequests().toString() =~ /[Vv]isionglass/) {
     apply plugin: 'com.huawei.agconnect'
 }
 
@@ -327,6 +328,7 @@ android {
                     arguments "-DVISIONGLASS=ON"
                 }
             }
+            buildConfigField "String", "HVR_API_KEY", "\"${getHVRApiKey()}\""
             buildConfigField "Float", "DEFAULT_DENSITY", "1.5f"
             buildConfigField "Float", "DEFAULT_WINDOW_DISTANCE", "1.0f"
         }
@@ -550,7 +552,8 @@ android {
 
         visionglass {
             java.srcDirs = [
-                    'src/visionglass/java'
+                    'src/visionglass/java',
+                    'src/hvr/java/com/igalia/wolvic/speech'
             ]
         }
 
@@ -722,6 +725,10 @@ dependencies {
 
     // Vission Glass
     visionglassImplementation fileTree(dir: "${project.rootDir}/third_party/aliceimu/", include: ['*.aar'])
+    visionglassImplementation fileTree(dir: "${project.rootDir}/third_party/hvr", include: ['*.jar'])
+    visionglassImplementation 'com.huawei.agconnect:agconnect-core-harmony:1.1.0.300'
+    visionglassImplementation 'com.huawei.agconnect:agconnect-core:1.9.1.301'
+    visionglassImplementation 'com.huawei.hms:ml-computer-voice-asr:3.1.0.300'
 
     // HTC Vive
     if (!gradle.startParameter.taskNames.isEmpty() &&
@@ -806,21 +813,21 @@ android.applicationVariants.configureEach { variant ->
         buildConfigField 'String', 'MLS_TOKEN', '""'
     }
 
-    // HVR packages for mainland china must only use HVR speech recognition system.
     def platform = variant.productFlavors.get(0).name
     def store = variant.productFlavors.get(3).name
-    if (platform.toLowerCase().startsWith('hvr') && store == "mainlandChina") {
-        variant.buildConfigField "String[]", "SPEECH_SERVICES", "{ com.igalia.wolvic.speech.SpeechServices.HUAWEI_ASR }"
-    }
 
-    // Default homepages for China releases.
+    // Default homepages and voice recognition services for China releases.
+    // HVR packages for mainland china must only use HVR speech recognition system.
     if (store == "mainlandChina") {
         if (platform.toLowerCase().startsWith('hvr')) {
             variant.resValue 'string', 'HOMEPAGE_URL', '"https://wolvic.com/zh/start/index.html"'
+            variant.buildConfigField "String[]", "SPEECH_SERVICES", "{ com.igalia.wolvic.speech.SpeechServices.HUAWEI_ASR }"
         } else if (platform.toLowerCase().startsWith('visionglass')) {
             variant.resValue 'string', 'HOMEPAGE_URL', '"https://wolvic.com/zh/start/hvg.html"'
+            variant.buildConfigField "String[]", "SPEECH_SERVICES", "{ com.igalia.wolvic.speech.SpeechServices.HUAWEI_ASR }"
         } else if (platform.toLowerCase().startsWith('picoxr')) {
             variant.resValue 'string', 'HOMEPAGE_URL', '"https://wolvic.com/zh/start/pico.html"'
+            variant.buildConfigField "String[]", "SPEECH_SERVICES", "{ com.igalia.wolvic.speech.SpeechServices.MEETKAI }"
         }
     }
 

diff --git a/app/src/common/shared/com/igalia/wolvic/PlatformActivityPlugin.java b/app/src/common/shared/com/igalia/wolvic/PlatformActivityPlugin.java
@@ -7,7 +7,6 @@ public abstract class PlatformActivityPlugin {
     public interface PlatformActivityPluginListener {
         void onPlatformScrollEvent(float distanceX, float distanceY);
     }
-    abstract void onKeyboardVisibilityChange(boolean isVisible);
     abstract void onVideoAvailabilityChange();
     void registerListener(PlatformActivityPluginListener listener) {
         if (mListeners == null)

diff --git a/app/src/common/shared/com/igalia/wolvic/VRBrowserActivity.java b/app/src/common/shared/com/igalia/wolvic/VRBrowserActivity.java
@@ -1710,10 +1710,6 @@ public void updateWidget(final Widget aWidget) {
             view.setVisibility(visible ? View.VISIBLE : View.GONE);
         }
 
-        if (aWidget == mKeyboard && mPlatformPlugin != null) {
-            mPlatformPlugin.onKeyboardVisibilityChange(visible);
-        }
-
         for (UpdateListener listener: mWidgetUpdateListeners) {
             listener.onWidgetUpdate(aWidget);
         }
@@ -1934,8 +1930,6 @@ public void setControllersVisible(final boolean aVisible) {
     @Override
     public void keyboardDismissed() {
         mNavigationBar.showVoiceSearch();
-        if (mPlatformPlugin != null)
-            mPlatformPlugin.onKeyboardVisibilityChange(false);
     }
 
     @Override

diff --git a/app/src/visionglass/java/com/igalia/wolvic/PlatformActivity.java b/app/src/visionglass/java/com/igalia/wolvic/PlatformActivity.java
@@ -28,6 +28,7 @@
 import android.view.KeyEvent;
 import android.view.LayoutInflater;
 import android.view.MotionEvent;
+import android.view.View;
 import android.view.WindowManager;
 import android.widget.Button;
 import android.widget.SeekBar;
@@ -39,6 +40,7 @@
 import androidx.fragment.app.FragmentActivity;
 import androidx.lifecycle.ViewModelProvider;
 
+import com.huawei.hms.mlsdk.common.MLApplication;
 import com.huawei.usblib.DisplayMode;
 import com.huawei.usblib.DisplayModeCallback;
 import com.huawei.usblib.OnConnectionListener;
@@ -48,8 +50,11 @@
 import com.igalia.wolvic.browser.api.WMediaSession;
 import com.igalia.wolvic.browser.api.WSession;
 import com.igalia.wolvic.databinding.VisionglassLayoutBinding;
+import com.igalia.wolvic.speech.SpeechRecognizer;
+import com.igalia.wolvic.speech.SpeechServices;
 import com.igalia.wolvic.ui.widgets.UIWidget;
 import com.igalia.wolvic.ui.widgets.WidgetManagerDelegate;
+import com.igalia.wolvic.utils.StringUtils;
 import com.igalia.wolvic.utils.SystemUtils;
 
 import java.util.ArrayList;
@@ -156,6 +161,8 @@ protected void onCreate(Bundle savedInstanceState) {
         usbPermissionFilter.addAction(HUAWEI_USB_PERMISSION);
         registerReceiver(mUsbPermissionReceiver, usbPermissionFilter);
 
+        initializeAGConnect();
+
         initVisionGlassPhoneUI();
 
         mDisplayManager.registerDisplayListener(mDisplayListener, null);
@@ -245,8 +252,6 @@ private void initVisionGlassPhoneUI() {
 
         mBinding.realignButton.setOnClickListener(v -> reorientController());
 
-        mBinding.voiceSearchButton.setEnabled(false);
-
         Button backButton = findViewById(R.id.back_button);
         backButton.setOnClickListener(v -> onBackPressed());
 
@@ -503,6 +508,27 @@ private void updateDisplays() {
         mViewModel.updateConnectionState(PhoneUIViewModel.ConnectionState.DISPLAY_UNAVAILABLE);
     }
 
+    private void initializeAGConnect() {
+        Log.e(LOGTAG, "initializeAGConnect");
+        try {
+            String speechService = SettingsStore.getInstance(this).getVoiceSearchService();
+            if (SpeechServices.HUAWEI_ASR.equals(speechService) && StringUtils.isEmpty(BuildConfig.HVR_API_KEY)) {
+                Log.e(LOGTAG, "HVR API key is not available");
+                return;
+            }
+            MLApplication.getInstance().setApiKey(BuildConfig.HVR_API_KEY);
+            try {
+                SpeechRecognizer speechRecognizer = SpeechServices.getInstance(this, speechService);
+                ((VRBrowserApplication) getApplicationContext()).setSpeechRecognizer(speechRecognizer);
+            } catch (Exception e) {
+                Log.e(LOGTAG, "Exception creating the speech recognizer: " + e);
+                ((VRBrowserApplication) getApplicationContext()).setSpeechRecognizer(null);
+            }
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
+
     public final PlatformActivityPlugin createPlatformPlugin(WidgetManagerDelegate delegate) {
         return new PlatformActivityPluginVisionGlass(delegate);
     }
@@ -517,11 +543,6 @@ private class PlatformActivityPluginVisionGlass extends PlatformActivityPlugin {
             setupPhoneUI();
         }
 
-        @Override
-        public void onKeyboardVisibilityChange(boolean isVisible) {
-            mBinding.voiceSearchButton.setEnabled(isVisible);
-        }
-
         @Override
         public void onVideoAvailabilityChange() {
             boolean isAvailable = mDelegate.getWindows().isVideoAvailable();
@@ -601,9 +622,11 @@ private void setupPhoneUI() {
             });
 
             mBinding.voiceSearchButton.setOnClickListener(v -> {
-                // Delegate all the voice input handling in the KeyboardWidget which already handles
-                // all the potential voice input cases.
-                mDelegate.getKeyboard().simulateVoiceButtonClick();
+                if (mDelegate.getKeyboard().getVisibility() == View.VISIBLE) {
+                    mDelegate.getKeyboard().simulateVoiceButtonClick();
+                } else {
+                    mDelegate.getNavigationBar().onVoiceSearchClicked();
+                }
             });
 
             mBinding.muteButton.setOnClickListener(v -> {