Android Embedding PR 13: Integrated text input, keyevent input, and some other channel comms in FlutterView. (#7979)

Author: matthew-carroll

shell/platform/android/io/flutter/embedding/engine/android/FlutterView.java

@@ -5,14 +5,31 @@
 package io.flutter.embedding.engine.android;
 
 import android.content.Context;
+import android.content.res.Configuration;
+import android.graphics.Rect;
+import android.os.Build;
+import android.os.LocaleList;
 import android.support.annotation.NonNull;
 import android.support.annotation.Nullable;
+import android.text.format.DateFormat;
 import android.util.AttributeSet;
 import android.util.Log;
+import android.view.KeyEvent;
+import android.view.MotionEvent;
+import android.view.WindowInsets;
+import android.view.inputmethod.EditorInfo;
+import android.view.inputmethod.InputConnection;
+import android.view.inputmethod.InputMethod;
+import android.view.inputmethod.InputMethodManager;
 import android.widget.FrameLayout;
 
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Locale;
+
 import io.flutter.embedding.engine.FlutterEngine;
 import io.flutter.embedding.engine.renderer.FlutterRenderer;
+import io.flutter.plugin.editing.TextInputPlugin;
 
 /**
  * Displays a Flutter UI on an Android device.
@@ -50,6 +67,16 @@ public class FlutterView extends FrameLayout {
   @Nullable
   private FlutterEngine flutterEngine;
 
+  // Components that process various types of Android View input and events,
+  // possibly storing intermediate state, and communicating those events to Flutter.
+  //
+  // These components essentially add some additional behavioral logic on top of
+  // existing, stateless system channels, e.g., KeyEventChannel, TextInputChannel, etc.
+  @Nullable
+  private TextInputPlugin textInputPlugin;
+  @Nullable
+  private AndroidKeyProcessor androidKeyProcessor;
+
   /**
    * Constructs a {@code FlutterSurfaceView} programmatically, without any XML attributes.
    *
@@ -103,6 +130,176 @@ private void init() {
     }
   }
 
+  //------- Start: Process View configuration that Flutter cares about. ------
+  /**
+   * Sends relevant configuration data from Android to Flutter when the Android
+   * {@link Configuration} changes.
+   *
+   * The Android {@link Configuration} might change as a result of device orientation
+   * change, device language change, device text scale factor change, etc.
+   */
+  @Override
+  protected void onConfigurationChanged(Configuration newConfig) {
+    super.onConfigurationChanged(newConfig);
+    sendLocalesToFlutter(newConfig);
+    sendUserSettingsToFlutter();
+  }
+
+  /**
+   * Invoked when this {@code FlutterView} changes size, including upon initial
+   * measure.
+   *
+   * The initial measure reports an {@code oldWidth} and {@code oldHeight} of zero.
+   *
+   * Flutter cares about the width and height of the view that displays it on the host
+   * platform. Therefore, when this method is invoked, the new width and height are
+   * communicated to Flutter as the "physical size" of the view that displays Flutter's
+   * UI.
+   */
+  @Override
+  protected void onSizeChanged(int width, int height, int oldWidth, int oldHeight) {
+    // TODO(mattcarroll): hookup to viewport metrics.
+    super.onSizeChanged(width, height, oldWidth, oldHeight);
+  }
+
+  /**
+   * Invoked when Android's desired window insets change, i.e., padding.
+   *
+   * Flutter does not use a standard {@code View} hierarchy and therefore Flutter is
+   * unaware of these insets. Therefore, this method calculates the viewport metrics
+   * that Flutter should use and then sends those metrics to Flutter.
+   *
+   * This callback is not present in API < 20, which means lower API devices will see
+   * the wider than expected padding when the status and navigation bars are hidden.
+   */
+  @Override
+  public final WindowInsets onApplyWindowInsets(WindowInsets insets) {
+    // TODO(mattcarroll): hookup to Flutter metrics.
+    return insets;
+  }
+
+  /**
+   * Invoked when Android's desired window insets change, i.e., padding.
+   *
+   * {@code fitSystemWindows} is an earlier version of
+   * {@link #onApplyWindowInsets(WindowInsets)}. See that method for more details
+   * about how window insets relate to Flutter.
+   */
+  @Override
+  @SuppressWarnings("deprecation")
+  protected boolean fitSystemWindows(Rect insets) {
+    // TODO(mattcarroll): hookup to Flutter metrics.
+    return super.fitSystemWindows(insets);
+  }
+  //------- End: Process View configuration that Flutter cares about. --------
+
+  //-------- Start: Process UI I/O that Flutter cares about. -------
+  /**
+   * Creates an {@link InputConnection} to work with a {@link android.view.inputmethod.InputMethodManager}.
+   *
+   * Any {@code View} that can take focus or process text input must implement this
+   * method by returning a non-null {@code InputConnection}. Flutter may render one or
+   * many focusable and text-input widgets, therefore {@code FlutterView} must support
+   * an {@code InputConnection}.
+   *
+   * The {@code InputConnection} returned from this method comes from a
+   * {@link TextInputPlugin}, which is owned by this {@code FlutterView}. A
+   * {@link TextInputPlugin} exists to encapsulate the nuances of input communication,
+   * rather than spread that logic throughout this {@code FlutterView}.
+   */
+  @Override
+  public InputConnection onCreateInputConnection(EditorInfo outAttrs) {
+    if (!isAttachedToFlutterEngine()) {
+      return super.onCreateInputConnection(outAttrs);
+    }
+
+    return textInputPlugin.createInputConnection(this, outAttrs);
+  }
+
+  /**
+   * Invoked when key is released.
+   *
+   * This method is typically invoked in response to the release of a physical
+   * keyboard key or a D-pad button. It is generally not invoked when a virtual
+   * software keyboard is used, though a software keyboard may choose to invoke
+   * this method in some situations.
+   *
+   * {@link KeyEvent}s are sent from Android to Flutter. {@link AndroidKeyProcessor}
+   * may do some additional work with the given {@link KeyEvent}, e.g., combine this
+   * {@code keyCode} with the previous {@code keyCode} to generate a unicode combined
+   * character.
+   */
+  @Override
+  public boolean onKeyUp(int keyCode, KeyEvent event) {
+    if (!isAttachedToFlutterEngine()) {
+      return super.onKeyUp(keyCode, event);
+    }
+
+    androidKeyProcessor.onKeyUp(event);
+    return super.onKeyUp(keyCode, event);
+  }
+
+  /**
+   * Invoked when key is pressed.
+   *
+   * This method is typically invoked in response to the press of a physical
+   * keyboard key or a D-pad button. It is generally not invoked when a virtual
+   * software keyboard is used, though a software keyboard may choose to invoke
+   * this method in some situations.
+   *
+   * {@link KeyEvent}s are sent from Android to Flutter. {@link AndroidKeyProcessor}
+   * may do some additional work with the given {@link KeyEvent}, e.g., combine this
+   * {@code keyCode} with the previous {@code keyCode} to generate a unicode combined
+   * character.
+   */
+  @Override
+  public boolean onKeyDown(int keyCode, KeyEvent event) {
+    if (!isAttachedToFlutterEngine()) {
+      return super.onKeyDown(keyCode, event);
+    }
+
+    androidKeyProcessor.onKeyDown(event);
+    return super.onKeyDown(keyCode, event);
+  }
+
+  /**
+   * Invoked by Android when a user touch event occurs.
+   *
+   * Flutter handles all of its own gesture detection and processing, therefore this
+   * method forwards all {@link MotionEvent} data from Android to Flutter.
+   */
+  @Override
+  public boolean onTouchEvent(MotionEvent event) {
+    if (!isAttachedToFlutterEngine()) {
+      return false;
+    }
+
+    // TODO(mattcarroll): forward event to touch processore when it's merged in.
+    return false;
+  }
+
+  /**
+   * Invoked by Android when a hover-compliant input system causes a hover event.
+   *
+   * An example of hover events is a stylus sitting near an Android screen. As the
+   * stylus moves from outside a {@code View} to hover over a {@code View}, or move
+   * around within a {@code View}, or moves from over a {@code View} to outside a
+   * {@code View}, a corresponding {@link MotionEvent} is reported via this method.
+   *
+   * Hover events can be used for accessibility touch exploration and therefore are
+   * processed here for accessibility purposes.
+   */
+  @Override
+  public boolean onHoverEvent(MotionEvent event) {
+    if (!isAttachedToFlutterEngine()) {
+      return false;
+    }
+
+    // TODO(mattcarroll): hook up to accessibility.
+    return false;
+  }
+  //-------- End: Process UI I/O that Flutter cares about. ---------
+
   /**
    * Connects this {@code FlutterView} to the given {@link FlutterEngine}.
    *
@@ -129,6 +326,26 @@ public void attachToFlutterEngine(@NonNull FlutterEngine flutterEngine) {
 
     // Instruct our FlutterRenderer that we are now its designated RenderSurface.
     this.flutterEngine.getRenderer().attachToRenderSurface(renderSurface);
+
+    // Initialize various components that know how to process Android View I/O
+    // in a way that Flutter understands.
+    textInputPlugin = new TextInputPlugin(
+        this,
+        this.flutterEngine.getDartExecutor()
+    );
+    androidKeyProcessor = new AndroidKeyProcessor(
+        this.flutterEngine.getKeyEventChannel(),
+        textInputPlugin
+    );
+
+    // Inform the Android framework that it should retrieve a new InputConnection
+    // now that an engine is attached.
+    // TODO(mattcarroll): once this is proven to work, move this line ot TextInputPlugin
+    textInputPlugin.getInputMethodManager().restartInput(this);
+
+    // Push View and Context related information from Android to Flutter.
+    sendUserSettingsToFlutter();
+    sendLocalesToFlutter(getResources().getConfiguration());
   }
 
   /**
@@ -147,6 +364,12 @@ public void detachFromFlutterEngine() {
     }
     Log.d(TAG, "Detaching from Flutter Engine");
 
+    // Inform the Android framework that it should retrieve a new InputConnection
+    // now that the engine is detached. The new InputConnection will be null, which
+    // signifies that this View does not process input (until a new engine is attached).
+    // TODO(mattcarroll): once this is proven to work, move this line ot TextInputPlugin
+    textInputPlugin.getInputMethodManager().restartInput(this);
+
     // Instruct our FlutterRenderer that we are no longer interested in being its RenderSurface.
     flutterEngine.getRenderer().detachFromRenderSurface();
     flutterEngine = null;
@@ -163,6 +386,42 @@ private boolean isAttachedToFlutterEngine() {
     return flutterEngine != null;
   }
 
+  /**
+   * Send the current {@link Locale} configuration to Flutter.
+   *
+   * FlutterEngine must be non-null when this method is invoked.
+   */
+  @SuppressWarnings("deprecation")
+  private void sendLocalesToFlutter(Configuration config) {
+    List<Locale> locales = new ArrayList<>();
+    if (Build.VERSION.SDK_INT >= android.os.Build.VERSION_CODES.N) {
+      LocaleList localeList = config.getLocales();
+      int localeCount = localeList.size();
+      for (int index = 0; index < localeCount; ++index) {
+        Locale locale = localeList.get(index);
+        locales.add(locale);
+      }
+    } else {
+      locales.add(config.locale);
+    }
+    flutterEngine.getLocalizationChannel().sendLocales(locales);
+  }
+
+  /**
+   * Send various user preferences of this Android device to Flutter.
+   *
+   * For example, sends the user's "text scale factor" preferences, as well as the user's clock
+   * format preference.
+   *
+   * FlutterEngine must be non-null when this method is invoked.
+   */
+  private void sendUserSettingsToFlutter() {
+    flutterEngine.getSettingsChannel().startMessage()
+        .setTextScaleFactor(getResources().getConfiguration().fontScale)
+        .setUse24HourFormat(DateFormat.is24HourFormat(getContext()))
+        .send();
+  }
+
   /**
    * Render modes for a {@link FlutterView}.
    */

shell/platform/android/io/flutter/plugin/editing/InputConnectionAdaptor.java

@@ -8,17 +8,17 @@
 import android.text.Editable;
 import android.text.Selection;
 import android.view.KeyEvent;
+import android.view.View;
 import android.view.inputmethod.BaseInputConnection;
 import android.view.inputmethod.EditorInfo;
 import android.view.inputmethod.InputMethodManager;
 
 import io.flutter.embedding.engine.systemchannels.TextInputChannel;
 import io.flutter.plugin.common.ErrorLogResult;
 import io.flutter.plugin.common.MethodChannel;
-import io.flutter.view.FlutterView;
 
 class InputConnectionAdaptor extends BaseInputConnection {
-    private final FlutterView mFlutterView;
+    private final View mFlutterView;
     private final int mClient;
     private final TextInputChannel textInputChannel;
     private final Editable mEditable;
@@ -29,7 +29,7 @@ class InputConnectionAdaptor extends BaseInputConnection {
         new ErrorLogResult("FlutterTextInput");
 
     public InputConnectionAdaptor(
-        FlutterView view,
+        View view,
         int client,
         TextInputChannel textInputChannel,
         Editable editable

shell/platform/android/io/flutter/plugin/editing/TextInputPlugin.java

@@ -10,6 +10,7 @@
 import android.text.Editable;
 import android.text.InputType;
 import android.text.Selection;
+import android.view.View;
 import android.view.inputmethod.BaseInputConnection;
 import android.view.inputmethod.EditorInfo;
 import android.view.inputmethod.InputConnection;
@@ -24,7 +25,7 @@
  */
 public class TextInputPlugin {
     @NonNull
-    private final FlutterView mView;
+    private final View mView;
     @NonNull
     private final InputMethodManager mImm;
     @NonNull
@@ -38,7 +39,7 @@ public class TextInputPlugin {
     @Nullable
     private InputConnection lastInputConnection;
 
-    public TextInputPlugin(FlutterView view, @NonNull DartExecutor dartExecutor) {
+    public TextInputPlugin(View view, @NonNull DartExecutor dartExecutor) {
         mView = view;
         mImm = (InputMethodManager) view.getContext().getSystemService(
                 Context.INPUT_METHOD_SERVICE);
@@ -126,7 +127,7 @@ private static int inputTypeFromTextInputType(
         return textType;
     }
 
-    public InputConnection createInputConnection(FlutterView view, EditorInfo outAttrs) {
+    public InputConnection createInputConnection(View view, EditorInfo outAttrs) {
         if (mClient == 0) {
             lastInputConnection = null;
             return lastInputConnection;
@@ -173,12 +174,12 @@ public InputConnection getLastInputConnection() {
         return lastInputConnection;
     }
 
-    private void showTextInput(FlutterView view) {
+    private void showTextInput(View view) {
         view.requestFocus();
         mImm.showSoftInput(view, 0);
     }
 
-    private void hideTextInput(FlutterView view) {
+    private void hideTextInput(View view) {
         mImm.hideSoftInputFromWindow(view.getApplicationWindowToken(), 0);
     }
 
@@ -203,7 +204,7 @@ private void applyStateToSelection(TextInputChannel.TextEditState state) {
         }
     }
 
-    private void setTextInputEditingState(FlutterView view, TextInputChannel.TextEditState state) {
+    private void setTextInputEditingState(View view, TextInputChannel.TextEditState state) {
         if (!mRestartInputPending && state.text.equals(mEditable.toString())) {
             applyStateToSelection(state);
             mImm.updateSelection(mView, Math.max(Selection.getSelectionStart(mEditable), 0),