From a036b74f79a0295b8ea3a1ece2d00ece9f05eb91 Mon Sep 17 00:00:00 2001 From: Felipe Erias Date: Fri, 14 Jun 2024 11:56:21 +0900 Subject: [PATCH] [Vision Glass] Voice button can be used for search and text input Set up the connection to the Huawei Speech Recognition service. The voice button on the phone UI will perform a search if the user is not currently writing on a text field (and therefore the keyboard is not visible). When the keyboard is visible, the voice button will capture text which will be entered into the text field that the user is editing. --- app/build.gradle | 23 ++++++---- .../igalia/wolvic/PlatformActivityPlugin.java | 1 - .../com/igalia/wolvic/VRBrowserActivity.java | 6 --- .../com/igalia/wolvic/PlatformActivity.java | 43 ++++++++++++++----- 4 files changed, 48 insertions(+), 25 deletions(-) diff --git a/app/build.gradle b/app/build.gradle index 1eab07832f..5227f1863d 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -9,8 +9,9 @@ apply from: "$project.rootDir/tools/gradle/versionCode.gradle" apply plugin: 'kotlin-android' apply plugin: "org.mozilla.telemetry.glean-gradle-plugin" -// Apply AGConnect plugin only for Hvr builds -if (getGradle().getStartParameter().getTaskRequests().toString() =~ /(h|H)vr/) { +// Apply AGConnect plugin only for Huawei builds +if (getGradle().getStartParameter().getTaskRequests().toString() =~ /[Hh]vr/ + || getGradle().getStartParameter().getTaskRequests().toString() =~ /[Vv]isionglass/) { apply plugin: 'com.huawei.agconnect' } @@ -327,6 +328,7 @@ android { arguments "-DVISIONGLASS=ON" } } + buildConfigField "String", "HVR_API_KEY", "\"${getHVRApiKey()}\"" buildConfigField "Float", "DEFAULT_DENSITY", "1.5f" buildConfigField "Float", "DEFAULT_WINDOW_DISTANCE", "1.0f" } @@ -550,7 +552,8 @@ android { visionglass { java.srcDirs = [ - 'src/visionglass/java' + 'src/visionglass/java', + 'src/hvr/java/com/igalia/wolvic/speech' ] } @@ -722,6 +725,10 @@ dependencies { // Vission Glass visionglassImplementation fileTree(dir: "${project.rootDir}/third_party/aliceimu/", include: ['*.aar']) + visionglassImplementation fileTree(dir: "${project.rootDir}/third_party/hvr", include: ['*.jar']) + visionglassImplementation 'com.huawei.agconnect:agconnect-core-harmony:1.1.0.300' + visionglassImplementation 'com.huawei.agconnect:agconnect-core:1.9.1.301' + visionglassImplementation 'com.huawei.hms:ml-computer-voice-asr:3.1.0.300' // HTC Vive if (!gradle.startParameter.taskNames.isEmpty() && @@ -806,21 +813,21 @@ android.applicationVariants.configureEach { variant -> buildConfigField 'String', 'MLS_TOKEN', '""' } - // HVR packages for mainland china must only use HVR speech recognition system. def platform = variant.productFlavors.get(0).name def store = variant.productFlavors.get(3).name - if (platform.toLowerCase().startsWith('hvr') && store == "mainlandChina") { - variant.buildConfigField "String[]", "SPEECH_SERVICES", "{ com.igalia.wolvic.speech.SpeechServices.HUAWEI_ASR }" - } - // Default homepages for China releases. + // Default homepages and voice recognition services for China releases. + // HVR packages for mainland china must only use HVR speech recognition system. if (store == "mainlandChina") { if (platform.toLowerCase().startsWith('hvr')) { variant.resValue 'string', 'HOMEPAGE_URL', '"https://wolvic.com/zh/start/index.html"' + variant.buildConfigField "String[]", "SPEECH_SERVICES", "{ com.igalia.wolvic.speech.SpeechServices.HUAWEI_ASR }" } else if (platform.toLowerCase().startsWith('visionglass')) { variant.resValue 'string', 'HOMEPAGE_URL', '"https://wolvic.com/zh/start/hvg.html"' + variant.buildConfigField "String[]", "SPEECH_SERVICES", "{ com.igalia.wolvic.speech.SpeechServices.HUAWEI_ASR }" } else if (platform.toLowerCase().startsWith('picoxr')) { variant.resValue 'string', 'HOMEPAGE_URL', '"https://wolvic.com/zh/start/pico.html"' + variant.buildConfigField "String[]", "SPEECH_SERVICES", "{ com.igalia.wolvic.speech.SpeechServices.MEETKAI }" } } diff --git a/app/src/common/shared/com/igalia/wolvic/PlatformActivityPlugin.java b/app/src/common/shared/com/igalia/wolvic/PlatformActivityPlugin.java index 0bcd61ed2c..9d08efd66d 100644 --- a/app/src/common/shared/com/igalia/wolvic/PlatformActivityPlugin.java +++ b/app/src/common/shared/com/igalia/wolvic/PlatformActivityPlugin.java @@ -7,7 +7,6 @@ public abstract class PlatformActivityPlugin { public interface PlatformActivityPluginListener { void onPlatformScrollEvent(float distanceX, float distanceY); } - abstract void onKeyboardVisibilityChange(boolean isVisible); abstract void onVideoAvailabilityChange(); void registerListener(PlatformActivityPluginListener listener) { if (mListeners == null) diff --git a/app/src/common/shared/com/igalia/wolvic/VRBrowserActivity.java b/app/src/common/shared/com/igalia/wolvic/VRBrowserActivity.java index 4fdbfc65da..4ff6570a1f 100644 --- a/app/src/common/shared/com/igalia/wolvic/VRBrowserActivity.java +++ b/app/src/common/shared/com/igalia/wolvic/VRBrowserActivity.java @@ -1710,10 +1710,6 @@ public void updateWidget(final Widget aWidget) { view.setVisibility(visible ? View.VISIBLE : View.GONE); } - if (aWidget == mKeyboard && mPlatformPlugin != null) { - mPlatformPlugin.onKeyboardVisibilityChange(visible); - } - for (UpdateListener listener: mWidgetUpdateListeners) { listener.onWidgetUpdate(aWidget); } @@ -1934,8 +1930,6 @@ public void setControllersVisible(final boolean aVisible) { @Override public void keyboardDismissed() { mNavigationBar.showVoiceSearch(); - if (mPlatformPlugin != null) - mPlatformPlugin.onKeyboardVisibilityChange(false); } @Override diff --git a/app/src/visionglass/java/com/igalia/wolvic/PlatformActivity.java b/app/src/visionglass/java/com/igalia/wolvic/PlatformActivity.java index 32f2e5a709..a1a5741e64 100644 --- a/app/src/visionglass/java/com/igalia/wolvic/PlatformActivity.java +++ b/app/src/visionglass/java/com/igalia/wolvic/PlatformActivity.java @@ -28,6 +28,7 @@ import android.view.KeyEvent; import android.view.LayoutInflater; import android.view.MotionEvent; +import android.view.View; import android.view.WindowManager; import android.widget.Button; import android.widget.SeekBar; @@ -39,6 +40,7 @@ import androidx.fragment.app.FragmentActivity; import androidx.lifecycle.ViewModelProvider; +import com.huawei.hms.mlsdk.common.MLApplication; import com.huawei.usblib.DisplayMode; import com.huawei.usblib.DisplayModeCallback; import com.huawei.usblib.OnConnectionListener; @@ -48,8 +50,11 @@ import com.igalia.wolvic.browser.api.WMediaSession; import com.igalia.wolvic.browser.api.WSession; import com.igalia.wolvic.databinding.VisionglassLayoutBinding; +import com.igalia.wolvic.speech.SpeechRecognizer; +import com.igalia.wolvic.speech.SpeechServices; import com.igalia.wolvic.ui.widgets.UIWidget; import com.igalia.wolvic.ui.widgets.WidgetManagerDelegate; +import com.igalia.wolvic.utils.StringUtils; import com.igalia.wolvic.utils.SystemUtils; import java.util.ArrayList; @@ -156,6 +161,8 @@ protected void onCreate(Bundle savedInstanceState) { usbPermissionFilter.addAction(HUAWEI_USB_PERMISSION); registerReceiver(mUsbPermissionReceiver, usbPermissionFilter); + initializeAGConnect(); + initVisionGlassPhoneUI(); mDisplayManager.registerDisplayListener(mDisplayListener, null); @@ -245,8 +252,6 @@ private void initVisionGlassPhoneUI() { mBinding.realignButton.setOnClickListener(v -> reorientController()); - mBinding.voiceSearchButton.setEnabled(false); - Button backButton = findViewById(R.id.back_button); backButton.setOnClickListener(v -> onBackPressed()); @@ -503,6 +508,27 @@ private void updateDisplays() { mViewModel.updateConnectionState(PhoneUIViewModel.ConnectionState.DISPLAY_UNAVAILABLE); } + private void initializeAGConnect() { + Log.e(LOGTAG, "initializeAGConnect"); + try { + String speechService = SettingsStore.getInstance(this).getVoiceSearchService(); + if (SpeechServices.HUAWEI_ASR.equals(speechService) && StringUtils.isEmpty(BuildConfig.HVR_API_KEY)) { + Log.e(LOGTAG, "HVR API key is not available"); + return; + } + MLApplication.getInstance().setApiKey(BuildConfig.HVR_API_KEY); + try { + SpeechRecognizer speechRecognizer = SpeechServices.getInstance(this, speechService); + ((VRBrowserApplication) getApplicationContext()).setSpeechRecognizer(speechRecognizer); + } catch (Exception e) { + Log.e(LOGTAG, "Exception creating the speech recognizer: " + e); + ((VRBrowserApplication) getApplicationContext()).setSpeechRecognizer(null); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + public final PlatformActivityPlugin createPlatformPlugin(WidgetManagerDelegate delegate) { return new PlatformActivityPluginVisionGlass(delegate); } @@ -517,11 +543,6 @@ private class PlatformActivityPluginVisionGlass extends PlatformActivityPlugin { setupPhoneUI(); } - @Override - public void onKeyboardVisibilityChange(boolean isVisible) { - mBinding.voiceSearchButton.setEnabled(isVisible); - } - @Override public void onVideoAvailabilityChange() { boolean isAvailable = mDelegate.getWindows().isVideoAvailable(); @@ -601,9 +622,11 @@ private void setupPhoneUI() { }); mBinding.voiceSearchButton.setOnClickListener(v -> { - // Delegate all the voice input handling in the KeyboardWidget which already handles - // all the potential voice input cases. - mDelegate.getKeyboard().simulateVoiceButtonClick(); + if (mDelegate.getKeyboard().getVisibility() == View.VISIBLE) { + mDelegate.getKeyboard().simulateVoiceButtonClick(); + } else { + mDelegate.getNavigationBar().onVoiceSearchClicked(); + } }); mBinding.muteButton.setOnClickListener(v -> {