Scribe (Android stylus handwriting text input)

shell/platform/android/BUILD.gn

@@ -284,6 +284,7 @@ android_java_sources = [
   "io/flutter/embedding/engine/systemchannels/PlatformViewsChannel.java",
   "io/flutter/embedding/engine/systemchannels/ProcessTextChannel.java",
   "io/flutter/embedding/engine/systemchannels/RestorationChannel.java",
+  "io/flutter/embedding/engine/systemchannels/ScribeChannel.java",
   "io/flutter/embedding/engine/systemchannels/SettingsChannel.java",
   "io/flutter/embedding/engine/systemchannels/SpellCheckChannel.java",
   "io/flutter/embedding/engine/systemchannels/SystemChannel.java",
@@ -310,6 +311,7 @@ android_java_sources = [
   "io/flutter/plugin/editing/ImeSyncDeferringInsetsCallback.java",
   "io/flutter/plugin/editing/InputConnectionAdaptor.java",
   "io/flutter/plugin/editing/ListenableEditingState.java",
+  "io/flutter/plugin/editing/ScribePlugin.java",
   "io/flutter/plugin/editing/SpellCheckPlugin.java",
   "io/flutter/plugin/editing/TextEditingDelta.java",
   "io/flutter/plugin/editing/TextInputPlugin.java",

shell/platform/android/io/flutter/embedding/android/FlutterView.java

@@ -39,6 +39,7 @@
 import android.view.inputmethod.InputConnection;
 import android.view.textservice.SpellCheckerInfo;
 import android.view.textservice.TextServicesManager;
+import android.view.InputDevice;
 import android.widget.FrameLayout;
 import androidx.annotation.NonNull;
 import androidx.annotation.Nullable;
@@ -62,6 +63,7 @@
 import io.flutter.embedding.engine.renderer.RenderSurface;
 import io.flutter.embedding.engine.systemchannels.SettingsChannel;
 import io.flutter.plugin.common.BinaryMessenger;
+import io.flutter.plugin.editing.ScribePlugin;
 import io.flutter.plugin.editing.SpellCheckPlugin;
 import io.flutter.plugin.editing.TextInputPlugin;
 import io.flutter.plugin.localization.LocalizationPlugin;
@@ -132,6 +134,7 @@ public class FlutterView extends FrameLayout
   @Nullable private MouseCursorPlugin mouseCursorPlugin;
   @Nullable private TextInputPlugin textInputPlugin;
   @Nullable private SpellCheckPlugin spellCheckPlugin;
+  @Nullable private ScribePlugin scribePlugin;
   @Nullable private LocalizationPlugin localizationPlugin;
   @Nullable private KeyboardManager keyboardManager;
   @Nullable private AndroidTouchProcessor androidTouchProcessor;
@@ -826,6 +829,20 @@ public InputConnection onCreateInputConnection(@NonNull EditorInfo outAttrs) {
     return textInputPlugin.createInputConnection(this, keyboardManager, outAttrs);
   }
 
+  @Override
+  public PointerIcon onResolvePointerIcon(MotionEvent event, int pointerIndex) {
+    // TODO(justinmc): Also need to check if over a valid field and if stylus
+    // input is supported.
+    // Maybe have to do this in the framework and show a Flutter icon?
+    final int toolType = event.getToolType(pointerIndex);
+    if (!event.isFromSource(InputDevice.SOURCE_MOUSE)
+      && event.isFromSource(InputDevice.SOURCE_STYLUS)
+      && toolType == MotionEvent.TOOL_TYPE_STYLUS) {
+      return PointerIcon.getSystemIcon(getContext(), PointerIcon.TYPE_HANDWRITING);
+    }
+    return super.onResolvePointerIcon(event, pointerIndex);
+  }
+
   /**
    * Allows a {@code View} that is not currently the input connection target to invoke commands on
    * the {@link android.view.inputmethod.InputMethodManager}, which is otherwise disallowed.
@@ -1103,10 +1120,13 @@ public void attachToFlutterEngine(@NonNull FlutterEngine flutterEngine) {
     if (Build.VERSION.SDK_INT >= API_LEVELS.API_24) {
       mouseCursorPlugin = new MouseCursorPlugin(this, this.flutterEngine.getMouseCursorChannel());
     }
+
     textInputPlugin =
         new TextInputPlugin(
             this,
             this.flutterEngine.getTextInputChannel(),
+            // TODO(justinmc): This could just be part of TextInputChannel...
+            this.flutterEngine.getScribeChannel(),
             this.flutterEngine.getPlatformViewsController());
 
     try {
@@ -1119,6 +1139,11 @@ public void attachToFlutterEngine(@NonNull FlutterEngine flutterEngine) {
       Log.e(TAG, "TextServicesManager not supported by device, spell check disabled.");
     }
 
+    if (Build.VERSION.SDK_INT >= API_LEVELS.API_34) {
+      scribePlugin =
+          new ScribePlugin(this, textInputPlugin.getInputMethodManager(), this.flutterEngine.getScribeChannel());
+    }
+
     localizationPlugin = this.flutterEngine.getLocalizationPlugin();
 
     keyboardManager = new KeyboardManager(this);

shell/platform/android/io/flutter/embedding/engine/FlutterEngine.java

@@ -34,6 +34,7 @@
 import io.flutter.embedding.engine.systemchannels.PlatformChannel;
 import io.flutter.embedding.engine.systemchannels.ProcessTextChannel;
 import io.flutter.embedding.engine.systemchannels.RestorationChannel;
+import io.flutter.embedding.engine.systemchannels.ScribeChannel;
 import io.flutter.embedding.engine.systemchannels.SettingsChannel;
 import io.flutter.embedding.engine.systemchannels.SpellCheckChannel;
 import io.flutter.embedding.engine.systemchannels.SystemChannel;
@@ -100,6 +101,7 @@ public class FlutterEngine implements ViewUtils.DisplayUpdater {
   @NonNull private final RestorationChannel restorationChannel;
   @NonNull private final PlatformChannel platformChannel;
   @NonNull private final ProcessTextChannel processTextChannel;
+  @NonNull private final ScribeChannel scribeChannel;
   @NonNull private final SettingsChannel settingsChannel;
   @NonNull private final SpellCheckChannel spellCheckChannel;
   @NonNull private final SystemChannel systemChannel;
@@ -337,6 +339,7 @@ public FlutterEngine(
     platformChannel = new PlatformChannel(dartExecutor);
     processTextChannel = new ProcessTextChannel(dartExecutor, context.getPackageManager());
     restorationChannel = new RestorationChannel(dartExecutor, waitForRestorationData);
+    scribeChannel = new ScribeChannel(dartExecutor);
     settingsChannel = new SettingsChannel(dartExecutor);
     spellCheckChannel = new SpellCheckChannel(dartExecutor);
     systemChannel = new SystemChannel(dartExecutor);
@@ -610,6 +613,12 @@ public TextInputChannel getTextInputChannel() {
     return textInputChannel;
   }
 
+  /** System channel that sends and receives Scribe requests and results. */
+  @NonNull
+  public ScribeChannel getScribeChannel() {
+    return scribeChannel;
+  }
+
   /** System channel that sends and receives spell check requests and results. */
   @NonNull
   public SpellCheckChannel getSpellCheckChannel() {

shell/platform/android/io/flutter/embedding/engine/systemchannels/ScribeChannel.java

@@ -0,0 +1,95 @@
+// Copyright 2013 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package io.flutter.embedding.engine.systemchannels;
+
+import android.graphics.RectF;
+import android.view.inputmethod.SelectGesture;
+import androidx.annotation.NonNull;
+import androidx.annotation.Nullable;
+import io.flutter.Log;
+import io.flutter.embedding.engine.dart.DartExecutor;
+import io.flutter.plugin.common.MethodCall;
+import io.flutter.plugin.common.MethodChannel;
+import io.flutter.plugin.common.StandardMethodCodec;
+import java.util.HashMap;
+import java.util.Arrays;
+
+/**
+ * {@link ScribeChannel} is a platform channel that is used by the framework to facilitate
+ * the Scribe handwriting text input feature.
+ */
+public class ScribeChannel {
+  private static final String TAG = "ScribeChannel";
+
+  public final MethodChannel channel;
+  private ScribeMethodHandler scribeMethodHandler;
+
+  @NonNull
+  public final MethodChannel.MethodCallHandler parsingMethodHandler =
+      new MethodChannel.MethodCallHandler() {
+        @Override
+        public void onMethodCall(@NonNull MethodCall call, @NonNull MethodChannel.Result result) {
+          if (scribeMethodHandler == null) {
+            Log.v(
+                TAG,
+                "No ScribeMethodHandler registered, call not forwarded to spell check API.");
+            return;
+          }
+          String method = call.method;
+          Object args = call.arguments;
+          Log.v(TAG, "Received '" + method + "' message.");
+          switch (method) {
+            case "Scribe.startStylusHandwriting":
+              try {
+                scribeMethodHandler.startStylusHandwriting();
+                result.success(null);
+              } catch (IllegalStateException exception) {
+                result.error("error", exception.getMessage(), null);
+              }
+              break;
+            default:
+              result.notImplemented();
+              break;
+          }
+        }
+      };
+
+  public ScribeChannel(@NonNull DartExecutor dartExecutor) {
+    channel = new MethodChannel(dartExecutor, "flutter/scribe", StandardMethodCodec.INSTANCE);
+    channel.setMethodCallHandler(parsingMethodHandler);
+  }
+
+  /**
+   * Sets the {@link ScribeMethodHandler} which receives all requests for scribe
+   * sent through this channel.
+   */
+  public void setScribeMethodHandler(
+      @Nullable ScribeMethodHandler scribeMethodHandler) {
+    this.scribeMethodHandler = scribeMethodHandler;
+  }
+
+  public interface ScribeMethodHandler {
+    /**
+     * Requests to start Scribe stylus handwriting, which will respond to the
+     * {@code result} with either success if handwriting input has started or
+     * error otherwise.
+     */
+    void startStylusHandwriting();
+  }
+
+  public void performHandwritingSelectGesture(SelectGesture gesture, MethodChannel.Result result) {
+    System.out.println("justin sending performSelectionGesture for gesture: " + gesture);
+    final HashMap<Object, Object> selectionAreaMap = new HashMap<>();
+    final RectF selectionArea = gesture.getSelectionArea();
+    selectionAreaMap.put("bottom", selectionArea.bottom);
+    selectionAreaMap.put("top", selectionArea.top);
+    selectionAreaMap.put("left", selectionArea.left);
+    selectionAreaMap.put("right", selectionArea.right);
+    // TODO(justinmc): Include granularity.
+    final HashMap<Object, Object> gestureMap = new HashMap<>();
+    gestureMap.put("selectionArea", selectionAreaMap);
+    channel.invokeMethod("ScribeClient.performSelectionGesture", Arrays.asList(gestureMap), result);
+  }
+}

shell/platform/android/io/flutter/plugin/editing/InputConnectionAdaptor.java

@@ -13,6 +13,7 @@
 import android.net.Uri;
 import android.os.Build;
 import android.os.Bundle;
+import android.os.CancellationSignal;
 import android.text.DynamicLayout;
 import android.text.Editable;
 import android.text.InputType;
@@ -26,18 +27,25 @@
 import android.view.inputmethod.EditorInfo;
 import android.view.inputmethod.ExtractedText;
 import android.view.inputmethod.ExtractedTextRequest;
+import android.view.inputmethod.HandwritingGesture;
 import android.view.inputmethod.InputContentInfo;
 import android.view.inputmethod.InputMethodManager;
+import android.view.inputmethod.PreviewableHandwritingGesture;
+import android.view.inputmethod.SelectGesture;
 import androidx.annotation.NonNull;
 import androidx.annotation.RequiresApi;
 import androidx.core.view.inputmethod.InputConnectionCompat;
 import io.flutter.Log;
 import io.flutter.embedding.engine.FlutterJNI;
+import io.flutter.embedding.engine.systemchannels.ScribeChannel;
 import io.flutter.embedding.engine.systemchannels.TextInputChannel;
+import io.flutter.plugin.common.MethodChannel;
 import java.io.ByteArrayOutputStream;
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.InputStream;
+import java.util.concurrent.Executor;
+import java.util.function.IntConsumer;
 import java.util.HashMap;
 import java.util.Map;
 
@@ -51,6 +59,7 @@ public interface KeyboardDelegate {
 
   private final View mFlutterView;
   private final int mClient;
+  private final ScribeChannel scribeChannel;
   private final TextInputChannel textInputChannel;
   private final ListenableEditingState mEditable;
   private final EditorInfo mEditorInfo;
@@ -69,6 +78,7 @@ public InputConnectionAdaptor(
       View view,
       int client,
       TextInputChannel textInputChannel,
+      ScribeChannel scribeChannel,
       KeyboardDelegate keyboardDelegate,
       ListenableEditingState editable,
       EditorInfo editorInfo,
@@ -77,6 +87,7 @@ public InputConnectionAdaptor(
     mFlutterView = view;
     mClient = client;
     this.textInputChannel = textInputChannel;
+    this.scribeChannel = scribeChannel;
     mEditable = editable;
     mEditable.addEditingStateListener(this);
     mEditorInfo = editorInfo;
@@ -100,10 +111,11 @@ public InputConnectionAdaptor(
       View view,
       int client,
       TextInputChannel textInputChannel,
+      ScribeChannel scribeChannel,
       KeyboardDelegate keyboardDelegate,
       ListenableEditingState editable,
       EditorInfo editorInfo) {
-    this(view, client, textInputChannel, keyboardDelegate, editable, editorInfo, new FlutterJNI());
+    this(view, client, textInputChannel, scribeChannel, keyboardDelegate, editable, editorInfo, new FlutterJNI());
   }
 
   private ExtractedText getExtractedText(ExtractedTextRequest request) {
@@ -262,6 +274,50 @@ public boolean setSelection(int start, int end) {
     return result;
   }
 
+  @Override
+  public boolean previewHandwritingGesture (PreviewableHandwritingGesture gesture,
+                CancellationSignal cancellationSignal) {
+    System.out.println("justin previewHandwritingGesture gesture: " + gesture);
+    return true;
+  }
+
+  @Override
+  public void performHandwritingGesture (HandwritingGesture gesture, Executor executor, IntConsumer consumer) {
+    System.out.println("justin performHandwritingGesture gesture: " + gesture);
+
+    if (gesture instanceof SelectGesture) {
+      final MethodChannel.Result result = new MethodChannel.Result() {
+        @Override
+        public void success(Object result) {
+          executor.execute(() -> consumer.accept(HANDWRITING_GESTURE_RESULT_SUCCESS));
+        }
+
+        @Override
+        public void error(String errorCode, String errorMessage, Object errorDetails) {
+          executor.execute(() -> consumer.accept(HANDWRITING_GESTURE_RESULT_FAILED));
+        }
+
+        @Override
+        public void notImplemented() {
+          executor.execute(() -> consumer.accept(HANDWRITING_GESTURE_RESULT_UNSUPPORTED));
+        }
+      };
+      scribeChannel.performHandwritingSelectGesture((SelectGesture) gesture, result);
+      return;
+    }
+
+    executor.execute(() -> consumer.accept(HANDWRITING_GESTURE_RESULT_UNSUPPORTED));
+    /*
+InputConnection#HANDWRITING_GESTURE_RESULT_SUCCESS
+InputConnection#HANDWRITING_GESTURE_RESULT_FAILED
+InputConnection#HANDWRITING_GESTURE_RESULT_FALLBACK
+The gesture is performed and fallback text is inserted.
+InputConnection#HANDWRITING_GESTURE_RESULT_UNSUPPORTED
+The gesture is not supported by the editor
+InputConnection#HANDWRITING_GESTURE_RESULT_CANCELLED
+    */
+  }
+
   // Sanitizes the index to ensure the index is within the range of the
   // contents of editable.
   private static int clampIndexToEditable(int index, Editable editable) {