Allow keyboard to be repositioned (#1466)

* Allow keyboard to be repositioned

* Do not save keyboard movement after app is restarted

* Address review comments

* Fix clamping issues and use correct max values based on movement anchor point

* Remove unused imports

app/CMakeLists.txt

@@ -41,6 +41,7 @@ add_library( # Sets the name of the library.
              src/main/cpp/VRLayer.cpp
              src/main/cpp/VRLayerNode.cpp
              src/main/cpp/Widget.cpp
+             src/main/cpp/WidgetMover.cpp
              src/main/cpp/WidgetPlacement.cpp
              src/main/cpp/WidgetResizer.cpp
            )

app/src/common/shared/org/mozilla/vrbrowser/VRBrowserActivity.java

@@ -713,6 +713,17 @@ void handleResize(final int aHandle, final float aWorldWidth, final float aWorld
         });
     }
 
+    @Keep
+    @SuppressWarnings("unused")
+    void handleMoveEnd(final int aHandle, final float aDeltaX, final float aDeltaY, final float aDeltaZ, final float aRotation) {
+        runOnUiThread(() -> {
+            Widget widget = mWidgets.get(aHandle);
+            if (widget != null) {
+                widget.handleMoveEvent(aDeltaX, aDeltaY, aDeltaZ, aRotation);
+            }
+        });
+    }
+
     @Keep
     @SuppressWarnings("unused")
     void registerExternalContext(long aContext) {
@@ -1017,6 +1028,16 @@ public void finishWidgetResize(final Widget aWidget) {
         queueRunnable(() -> finishWidgetResizeNative(aWidget.getHandle()));
     }
 
+    @Override
+    public void startWidgetMove(final Widget aWidget, @WidgetMoveBehaviourFlags int aMoveBehaviour) {
+        queueRunnable(() -> startWidgetMoveNative(aWidget.getHandle(), aMoveBehaviour));
+    }
+
+    @Override
+    public void finishWidgetMove() {
+        queueRunnable(() -> finishWidgetMoveNative());
+    }
+
     @Override
     public void addUpdateListener(UpdateListener aUpdateListener) {
         if (!mWidgetUpdateListeners.contains(aUpdateListener)) {
@@ -1215,6 +1236,8 @@ public void setCylinderDensity(final float aDensity) {
     private native void removeWidgetNative(int aHandle);
     private native void startWidgetResizeNative(int aHandle);
     private native void finishWidgetResizeNative(int aHandle);
+    private native void startWidgetMoveNative(int aHandle, int aMoveBehaviour);
+    private native void finishWidgetMoveNative();
     private native void setWorldBrightnessNative(float aBrigthness);
     private native void setTemporaryFilePath(String aPath);
     private native void exitImmersiveNative();

app/src/common/shared/org/mozilla/vrbrowser/browser/SettingsStore.java

@@ -8,7 +8,6 @@
 
 import org.mozilla.geckoview.GeckoSessionSettings;
 import org.mozilla.telemetry.TelemetryHolder;
-import org.mozilla.vrbrowser.BuildConfig;
 import org.mozilla.vrbrowser.R;
 import org.mozilla.vrbrowser.telemetry.TelemetryWrapper;
 import org.mozilla.vrbrowser.utils.DeviceType;

app/src/common/shared/org/mozilla/vrbrowser/ui/widgets/KeyboardWidget.java

@@ -15,6 +15,7 @@
 import android.text.TextWatcher;
 import android.util.AttributeSet;
 import android.util.Log;
+import android.view.MotionEvent;
 import android.view.View;
 import android.view.ViewGroup;
 import android.view.inputmethod.CursorAnchorInfo;
@@ -88,6 +89,7 @@ public class KeyboardWidget extends UIWidget implements CustomKeyboardView.OnKey
     private int mKeyWidth;
     private int mKeyboardPopupTopMargin;
     private ImageButton mCloseKeyboardButton;
+    private ImageButton mKeyboardMoveButton;
     private boolean mIsLongPress;
     private boolean mIsMultiTap;
     private boolean mIsCapsLock;
@@ -97,6 +99,43 @@ public class KeyboardWidget extends UIWidget implements CustomKeyboardView.OnKey
     private String mComposingDisplayText = "";
     private boolean mInternalDeleteHint = false;
 
+    private class MoveTouchListener implements OnTouchListener {
+        @Override
+        public boolean onTouch(View v, MotionEvent event) {
+            switch (event.getAction()) {
+                case MotionEvent.ACTION_POINTER_DOWN:
+                case MotionEvent.ACTION_DOWN:
+                    v.setPressed(true);
+                    mWidgetManager.startWidgetMove(KeyboardWidget.this, WidgetManagerDelegate.WIDGET_MOVE_BEHAVIOUR_KEYBOARD);
+                    break;
+                case MotionEvent.ACTION_POINTER_UP:
+                case MotionEvent.ACTION_UP:
+                case MotionEvent.ACTION_CANCEL:
+                    v.setPressed(false);
+                    mWidgetManager.finishWidgetMove();
+                    break;
+                default:
+                    return false;
+
+            }
+            return true;
+        }
+    }
+
+    private class MoveHoverListener implements OnHoverListener {
+        @Override
+        public boolean onHover(View v, MotionEvent event) {
+            switch (event.getAction()) {
+                case MotionEvent.ACTION_HOVER_ENTER:
+                    v.setHovered(true);
+                    break;
+                case MotionEvent.ACTION_HOVER_EXIT:
+                    v.setHovered(false);
+            }
+            return false;
+        }
+    }
+
     public KeyboardWidget(Context aContext) {
         super(aContext);
         initialize(aContext);
@@ -179,7 +218,9 @@ private void initialize(Context aContext) {
         mCapsLockOnIcon = getResources().getDrawable(R.drawable.ic_icon_keyboard_caps, getContext().getTheme());
         mCloseKeyboardButton = findViewById(R.id.keyboardCloseButton);
         mCloseKeyboardButton.setOnClickListener(v -> dismiss());
-
+        mKeyboardMoveButton = findViewById(R.id.keyboardMoveButton);
+        mKeyboardMoveButton.setOnTouchListener(new MoveTouchListener());
+        mKeyboardMoveButton.setOnHoverListener(new MoveHoverListener());
         mKeyWidth = getResources().getDimensionPixelSize(R.dimen.keyboard_key_width);
         mKeyboardPopupTopMargin  = getResources().getDimensionPixelSize(R.dimen.keyboard_key_pressed_padding) * 2;
 

app/src/common/shared/org/mozilla/vrbrowser/ui/widgets/UIWidget.java

@@ -172,6 +172,14 @@ public void handleResizeEvent(float aWorldWidth, float aWorldHeight) {
         mWidgetManager.updateWidget(this);
     }
 
+    @Override
+    public void handleMoveEvent(float aDeltaX, float aDeltaY, float aDeltaZ, float aRotation) {
+        mWidgetPlacement.translationX += aDeltaX;
+        mWidgetPlacement.translationY += aDeltaY;
+        mWidgetPlacement.translationZ += aDeltaZ;
+        mWidgetPlacement.rotation = aRotation;
+    }
+
     @Override
     public void releaseWidget() {
         if (mRenderer != null) {

app/src/common/shared/org/mozilla/vrbrowser/ui/widgets/Widget.java

@@ -20,6 +20,7 @@ public interface Widget {
     void handleTouchEvent(MotionEvent aEvent);
     void handleHoverEvent(MotionEvent aEvent);
     void handleResizeEvent(float aWorldWidth, float aWorldHeight);
+    void handleMoveEvent(float aDeltaX, float aDeltaY, float aDeltaZ, float aRotation);
     void releaseWidget();
     void setFirstDraw(boolean aIsFirstDraw);
     boolean getFirstDraw();

app/src/common/shared/org/mozilla/vrbrowser/ui/widgets/WidgetManagerDelegate.java

@@ -4,6 +4,7 @@
 
 import org.mozilla.geckoview.GeckoSession;
 
+import androidx.annotation.IntDef;
 import androidx.annotation.NonNull;
 
 public interface WidgetManagerDelegate {
@@ -23,12 +24,19 @@ interface WorldClickListener {
     float DEFAULT_DIM_BRIGHTNESS = 0.25f;
     float DEFAULT_NO_DIM_BRIGHTNESS = 1.0f;
 
+    @IntDef(value = { WIDGET_MOVE_BEHAVIOUR_GENERAL, WIDGET_MOVE_BEHAVIOUR_KEYBOARD})
+    public @interface WidgetMoveBehaviourFlags {}
+    public static final int WIDGET_MOVE_BEHAVIOUR_GENERAL = 0;
+    public static final int WIDGET_MOVE_BEHAVIOUR_KEYBOARD = 1;
+
     int newWidgetHandle();
     void addWidget(@NonNull Widget aWidget);
     void updateWidget(@NonNull Widget aWidget);
     void removeWidget(@NonNull Widget aWidget);
     void startWidgetResize(@NonNull Widget aWidget);
     void finishWidgetResize(@NonNull Widget aWidget);
+    void startWidgetMove(@NonNull Widget aWidget, @WidgetMoveBehaviourFlags int aMoveBehaviour);
+    void finishWidgetMove();
     void addUpdateListener(@NonNull UpdateListener aUpdateListener);
     void removeUpdateListener(@NonNull UpdateListener aUpdateListener);
     void pushBackHandler(@NonNull Runnable aRunnable);

app/src/main/cpp/BrowserWorld.cpp

@@ -17,6 +17,7 @@
 #include "SplashAnimation.h"
 #include "Pointer.h"
 #include "Widget.h"
+#include "WidgetMover.h"
 #include "WidgetPlacement.h"
 #include "Cylinder.h"
 #include "Quad.h"
@@ -73,7 +74,6 @@ const int GestureSwipeLeft = 0;
 const int GestureSwipeRight = 1;
 
 const float kScrollFactor = 20.0f; // Just picked what fell right.
-const float kWorldDPIRatio = 2.0f/720.0f;
 const double kHoverRate = 1.0 / 10.0;
 
 class SurfaceObserver;
@@ -181,6 +181,7 @@ struct BrowserWorld::State {
   SplashAnimationPtr splashAnimation;
   VRVideoPtr vrVideo;
   PerformanceMonitorPtr monitor;
+  WidgetMoverPtr movingWidget;
 
   State() : paused(true), glInitialized(false), modelsLoaded(false), env(nullptr), cylinderDensity(0.0f), nearClip(0.1f),
             farClip(300.0f), activity(nullptr), windowsInitialized(false), exitImmersiveRequested(false), loaderDelay(0) {
@@ -361,6 +362,7 @@ BrowserWorld::State::UpdateControllers(bool& aRelayoutWidgets) {
 
     vrb::Vector start = controller.transformMatrix.MultiplyPosition(controller.beamTransformMatrix.MultiplyPosition(vrb::Vector()));
     vrb::Vector direction = controller.transformMatrix.MultiplyDirection(controller.beamTransformMatrix.MultiplyDirection(vrb::Vector(0.0f, 0.0f, -1.0f)));
+
     WidgetPtr hitWidget;
     float hitDistance = farClip;
     vrb::Vector hitPoint;
@@ -370,7 +372,8 @@ BrowserWorld::State::UpdateControllers(bool& aRelayoutWidgets) {
       vrb::Vector normal;
       float distance = 0.0f;
       bool isInWidget = false;
-      if (widget->TestControllerIntersection(start, direction, result, normal, isInWidget, distance)) {
+      const bool clamp = !widget->IsResizing();
+      if (widget->TestControllerIntersection(start, direction, result, normal, clamp, isInWidget, distance)) {
         if (isInWidget && (distance < hitDistance)) {
           hitWidget = widget;
           hitDistance = distance;
@@ -401,7 +404,18 @@ BrowserWorld::State::UpdateControllers(bool& aRelayoutWidgets) {
                          controller.buttonState & ControllerDelegate::BUTTON_TOUCHPAD;
     const bool wasPressed = controller.lastButtonState & ControllerDelegate::BUTTON_TRIGGER ||
                               controller.lastButtonState & ControllerDelegate::BUTTON_TOUCHPAD;
-    if (hitWidget && hitWidget->IsResizing()) {
+
+    if (movingWidget && movingWidget->IsMoving(controller.index)) {
+      if (!pressed && wasPressed) {
+        movingWidget->EndMoving();
+      } else {
+        WidgetPlacementPtr updatedPlacement = movingWidget->HandleMove(start, direction);
+        if (updatedPlacement) {
+          movingWidget->GetWidget()->SetPlacement(updatedPlacement);
+          aRelayoutWidgets = true;
+        }
+      }
+    } else if (hitWidget && hitWidget->IsResizing()) {
       bool aResized = false, aResizeEnded = false;
       hitWidget->HandleResize(hitPoint, pressed, aResized, aResizeEnded);
 
@@ -428,6 +442,7 @@ BrowserWorld::State::UpdateControllers(bool& aRelayoutWidgets) {
       if (!pressed && wasPressed) {
         controller.inDeadZone = true;
       }
+      controller.pointerWorldPoint = hitPoint;
       const bool moved = pressed ? OutOfDeadZone(controller, theX, theY)
           : (controller.pointerX != theX) || (controller.pointerY != theY);
       const bool throttled = ThrottleHoverEvent(controller, context->GetTimestamp(), pressed, wasPressed);
@@ -818,7 +833,7 @@ BrowserWorld::AddWidget(int32_t aHandle, const WidgetPlacementPtr& aPlacement) {
   }
   float worldWidth = aPlacement->worldWidth;
   if (worldWidth <= 0.0f) {
-    worldWidth = aPlacement->width * kWorldDPIRatio;
+    worldWidth = aPlacement->width * WidgetPlacement::kWorldDPIRatio;
   }
 
   const int32_t textureWidth = aPlacement->GetTextureWidth();
@@ -880,7 +895,7 @@ BrowserWorld::UpdateWidget(int32_t aHandle, const WidgetPlacementPtr& aPlacement
 
   float newWorldWidth = aPlacement->worldWidth;
   if (newWorldWidth <= 0.0f) {
-    newWorldWidth = aPlacement->width * kWorldDPIRatio;
+    newWorldWidth = aPlacement->width * WidgetPlacement::kWorldDPIRatio;
   }
 
   if (newWorldWidth != worldWidth || oldWidth != aPlacement->width || oldHeight != aPlacement->height) {
@@ -926,6 +941,51 @@ BrowserWorld::FinishWidgetResize(int32_t aHandle) {
   widget->FinishResize();
 }
 
+void
+BrowserWorld::StartWidgetMove(int32_t aHandle, int32_t aMoveBehavour) {
+  ASSERT_ON_RENDER_THREAD();
+  WidgetPtr widget = m.GetWidget(aHandle);
+  if (!widget) {
+    return;
+  }
+
+  vrb::Vector initialPoint;
+  vrb::Vector anchorPoint;
+  int controllerIndex = -1;
+
+  for (Controller& controller: m.controllers->GetControllers()) {
+    if (!controller.enabled || (controller.index < 0)) {
+      continue;
+    }
+
+    if (controller.pointer && controller.pointer->GetHitWidget() == widget) {
+      controllerIndex = controller.index;
+      initialPoint = controller.pointerWorldPoint;
+      int32_t w, h;
+      widget->GetSurfaceTextureSize(w, h);
+      anchorPoint.x() = controller.pointerX / (float)w;
+      anchorPoint.y() = controller.pointerY / (float)h;
+      break;
+    }
+  }
+
+  if (controllerIndex < 0) {
+    return;
+  }
+
+  m.movingWidget = WidgetMover::Create();
+  m.movingWidget->StartMoving(widget, aMoveBehavour, controllerIndex, initialPoint, anchorPoint);
+}
+
+void
+BrowserWorld::FinishWidgetMove() {
+  ASSERT_ON_RENDER_THREAD();
+  if (m.movingWidget) {
+    m.movingWidget->EndMoving();
+  }
+  m.movingWidget = nullptr;
+}
+
 void
 BrowserWorld::UpdateVisibleWidgets() {
   ASSERT_ON_RENDER_THREAD();
@@ -956,9 +1016,9 @@ BrowserWorld::LayoutWidget(int32_t aHandle) {
 
   vrb::Matrix transform = vrb::Matrix::Identity();
 
-  vrb::Vector translation = vrb::Vector(aPlacement->translation.x() * kWorldDPIRatio,
-                                        aPlacement->translation.y() * kWorldDPIRatio,
-                                        aPlacement->translation.z() * kWorldDPIRatio);
+  vrb::Vector translation = vrb::Vector(aPlacement->translation.x() * WidgetPlacement::kWorldDPIRatio,
+                                        aPlacement->translation.y() * WidgetPlacement::kWorldDPIRatio,
+                                        aPlacement->translation.z() * WidgetPlacement::kWorldDPIRatio);
 
   const float anchorX = (aPlacement->anchor.x() - 0.5f) * worldWidth;
   const float anchorY = (aPlacement->anchor.y() - 0.5f) * worldHeight;
@@ -1356,6 +1416,16 @@ JNI_METHOD(void, finishWidgetResizeNative)
   crow::BrowserWorld::Instance().FinishWidgetResize(aHandle);
 }
 
+JNI_METHOD(void, startWidgetMoveNative)
+(JNIEnv*, jobject, jint aHandle, jint aMoveBehaviour) {
+  crow::BrowserWorld::Instance().StartWidgetMove(aHandle, aMoveBehaviour);
+}
+
+JNI_METHOD(void, finishWidgetMoveNative)
+(JNIEnv*, jobject) {
+  crow::BrowserWorld::Instance().FinishWidgetMove();
+}
+
 JNI_METHOD(void, setWorldBrightnessNative)
 (JNIEnv*, jobject, jfloat aBrightness) {
   crow::BrowserWorld::Instance().SetBrightness(aBrightness);

app/src/main/cpp/BrowserWorld.h

@@ -48,6 +48,8 @@ class BrowserWorld {
   void RemoveWidget(int32_t aHandle);
   void StartWidgetResize(int32_t aHandle);
   void FinishWidgetResize(int32_t aHandle);
+  void StartWidgetMove(int32_t aHandle, const int32_t aMoveBehavour);
+  void FinishWidgetMove();
   void UpdateVisibleWidgets();
   void LayoutWidget(int32_t aHandle);
   void SetBrightness(const float aBrightness);

app/src/main/cpp/Controller.cpp

@@ -33,6 +33,7 @@ Controller::operator=(const Controller& aController) {
   widget = aController.widget;
   pointerX = aController.pointerX;
   pointerY = aController.pointerY;
+  pointerWorldPoint = aController.pointerWorldPoint;
   buttonState = aController.buttonState;
   lastButtonState = aController.lastButtonState;
   touched = aController.touched;
@@ -70,6 +71,7 @@ Controller::Reset() {
   focused = false;
   widget = 0;
   pointerX = pointerY = 0.0f;
+  pointerWorldPoint = vrb::Vector(0.0f, 0.0f, 0.0f);
   buttonState = lastButtonState = 0;
   touched = wasTouched = false;
   touchX = touchY = 0.0f;

app/src/main/cpp/Controller.h

@@ -26,6 +26,7 @@ struct Controller {
   uint32_t widget;
   float pointerX;
   float pointerY;
+  vrb::Vector pointerWorldPoint;
   uint32_t buttonState;
   uint32_t lastButtonState;
   bool touched;