Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[WIP][iOS]Add TextField OCR engine side support #29229

Closed
wants to merge 21 commits into from
Closed
Show file tree
Hide file tree
Changes from 20 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions lib/ui/platform_dispatcher.dart
Original file line number Diff line number Diff line change
Expand Up @@ -860,6 +860,10 @@ class PlatformDispatcher {
bool get brieflyShowPassword => _brieflyShowPassword;
bool _brieflyShowPassword = true;

/// Indicates this device supports capture text from camera or not
bool get captureTextFromCameraEnabled => _captureTextFromCameraEnabled;
bool _captureTextFromCameraEnabled = false;

/// The setting indicating the current brightness mode of the host platform.
/// If the platform has no preference, [platformBrightness] defaults to
/// [Brightness.light].
Expand Down Expand Up @@ -922,6 +926,11 @@ class PlatformDispatcher {
if (brieflyShowPassword != null) {
_brieflyShowPassword = brieflyShowPassword;
}
// This field is optional.
final bool? captureTextFromCameraEnabled = data['captureTextFromCameraEnabled'] as bool?;
if (captureTextFromCameraEnabled != null) {
_captureTextFromCameraEnabled = captureTextFromCameraEnabled;
}
final Brightness platformBrightness =
data['platformBrightness']! as String == 'dark' ? Brightness.dark : Brightness.light;
final String? systemFontFamily = data['systemFontFamily'] as String?;
Expand Down
6 changes: 6 additions & 0 deletions lib/ui/window.dart
Original file line number Diff line number Diff line change
Expand Up @@ -491,6 +491,12 @@ class SingletonFlutterWindow extends FlutterWindow {
platformDispatcher.onPlatformBrightnessChanged = callback;
}


/// A property that indicates this device support capture text from camera (OCR) or not
///
/// If the device doesn't support OCR ,[captureTextFromCameraEnabled] is false
bool get captureTextFromCameraEnabled => platformDispatcher.captureTextFromCameraEnabled;

/// The setting indicating the system font of the host platform.
///
/// {@macro dart.ui.window.accessorForwardWarning}
Expand Down
2 changes: 2 additions & 0 deletions lib/web_ui/lib/platform_dispatcher.dart
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,8 @@ abstract class PlatformDispatcher {

bool get brieflyShowPassword => true;

bool get captureTextFromCameraEnabled => false;

VoidCallback? get onTextScaleFactorChanged;
set onTextScaleFactorChanged(VoidCallback? callback);

Expand Down
2 changes: 2 additions & 0 deletions lib/web_ui/lib/window.dart
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,8 @@ abstract class SingletonFlutterWindow extends FlutterWindow {

bool get brieflyShowPassword => platformDispatcher.brieflyShowPassword;

bool get captureTextFromCameraEnabled => platformDispatcher.captureTextFromCameraEnabled;

bool get alwaysUse24HourFormat => platformDispatcher.alwaysUse24HourFormat;

VoidCallback? get onTextScaleFactorChanged => platformDispatcher.onTextScaleFactorChanged;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,8 @@
@"TextInput.setEditableSizeAndTransform";
static NSString* const kSetMarkedTextRectMethod = @"TextInput.setMarkedTextRect";
static NSString* const kFinishAutofillContextMethod = @"TextInput.finishAutofillContext";
static NSString* const kStartCapturingTextFromCameraMethod =
@"TextInput.startCapturingTextFromCamera";
static NSString* const kSetSelectionRectsMethod = @"TextInput.setSelectionRects";

#pragma mark - TextInputConfiguration Field Names
Expand Down Expand Up @@ -2089,6 +2091,9 @@ - (void)handleMethodCall:(FlutterMethodCall*)call result:(FlutterResult)result {
} else if ([method isEqualToString:kSetSelectionRectsMethod]) {
[self setSelectionRects:args];
result(nil);
} else if ([method isEqualToString:kStartCapturingTextFromCameraMethod]) {
[self startCapturingTextFromCamera];
result(nil);
} else {
result(FlutterMethodNotImplemented);
}
Expand Down Expand Up @@ -2170,6 +2175,15 @@ - (void)hideTextInput {
[_inputHider removeFromSuperview];
}

- (void)startCapturingTextFromCamera {
if (_activeView == nil || !_activeView.isFirstResponder) {
return;
}
if (@available(iOS 15.0, *)) {
[_activeView captureTextFromCamera:nil];
}
}

- (void)triggerAutofillSave:(BOOL)saveEntries {
[_activeView resignFirstResponder];

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@ - (void)cleanUpViewHierarchy:(BOOL)includeActiveView
- (UIView*)hostView;
- (fml::WeakPtr<FlutterTextInputPlugin>)getWeakPtr;
- (void)addToInputParentViewIfNeeded:(FlutterTextInputView*)inputView;
- (void)startCapturingTextFromCamera;
@end

@interface FlutterTextInputPluginTest : XCTestCase
Expand Down Expand Up @@ -165,6 +166,18 @@ - (FlutterTextRange*)getLineRangeFromTokenizer:(id<UITextInputTokenizer>)tokeniz
}

#pragma mark - Tests

- (void)testInvokeStartCapturingTextFromCamera {
FlutterMethodCall* methodCall =
[FlutterMethodCall methodCallWithMethodName:@"TextInput.startCapturingTextFromCamera"
arguments:nil];
FlutterTextInputPlugin* mockPlugin = OCMPartialMock(textInputPlugin);
[mockPlugin handleMethodCall:methodCall
result:^(id _Nullable result){
}];
OCMVerify([mockPlugin startCapturingTextFromCamera]);
}

- (void)testNoDanglingEnginePointer {
__weak FlutterTextInputPlugin* weakFlutterTextInputPlugin;
FlutterViewController* flutterViewController = [FlutterViewController new];
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,10 @@ @implementation FlutterViewController {
fml::scoped_nsobject<UIPanGestureRecognizer> _panGestureRecognizer API_AVAILABLE(ios(13.4));
fml::scoped_nsobject<UIView> _keyboardAnimationView;
MouseState _mouseState;

// This textField is to judge whether this device supports caputureTextFromCamera API or not.
// See method "captureTextFromCameraEnabled"
fml::scoped_nsobject<UITextField> _textField;
}

@synthesize displayingFlutterUI = _displayingFlutterUI;
Expand Down Expand Up @@ -624,6 +628,14 @@ - (void)setSplashScreenView:(UIView*)view {
UIViewAutoresizingFlexibleWidth | UIViewAutoresizingFlexibleHeight;
}

- (UITextField*)textField {
if (_textField.get() == nil) {
UITextField* newTextField = [[UITextField alloc] init];
_textField.reset(newTextField);
}
return _textField.get();
}

- (void)setFlutterViewDidRenderCallback:(void (^)(void))callback {
_flutterViewRenderedCallback.reset(callback, fml::OwnershipPolicy::Retain);
}
Expand Down Expand Up @@ -1512,7 +1524,8 @@ - (void)onUserSettingsChanged:(NSNotification*)notification {
@"textScaleFactor" : @([self textScaleFactor]),
@"alwaysUse24HourFormat" : @([self isAlwaysUse24HourFormat]),
@"platformBrightness" : [self brightnessMode],
@"platformContrast" : [self contrastMode]
@"platformContrast" : [self contrastMode],
@"captureTextFromCameraEnabled" : @([self captureTextFromCameraEnabled]),
}];
}

Expand Down Expand Up @@ -1621,6 +1634,11 @@ - (NSString*)contrastMode {
}
}

- (BOOL)captureTextFromCameraEnabled {
UITextField* textField = [self textField];
return [textField canPerformAction:@selector(captureTextFromCamera:) withSender:nil];
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is there a notification you can listen to, if the value you get here changes?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

For instance will the value become false, if the user rejects the camera permission?

Copy link
Contributor Author

@luckysmg luckysmg Jun 1, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This api invokes system internal camera API and it seems that is will not ask users for camera permission..

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

iOSDemo 2.zip
Here is demo code. I didn't request any permission

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Without the capability of monitoring the results of this canPerformAction: call I think it would be better to implement this feature in a way that's similar to paste: introduce an API that allows the developer to query whether live text is currently enabled on the platform, otherwise they may get outdated value?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Got it. I will take a look. Thx

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ah I thought you could still turn camera access off in Settings? (don't have a test device available atm, I'll give this a try tomorrow)

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah if you disable camera access in Settings -> Screen Time -> Content & Privacy Restrictions -> Allowed Apps, the text edit menu item will be gone.

}

#pragma mark - Status bar style

- (UIStatusBarStyle)preferredStatusBarStyle {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -143,6 +143,7 @@ - (void)startKeyBoardAnimation:(NSTimeInterval)duration;
- (void)ensureViewportMetricsIsCorrect;
- (void)invalidateDisplayLink;
- (void)addInternalPlugins;
- (BOOL)captureTextFromCameraEnabled;
- (flutter::PointerData)generatePointerDataForFake;
@end

Expand Down Expand Up @@ -171,6 +172,18 @@ - (void)tearDown {
self.messageSent = nil;
}

- (void)testOnUserSettingsChangedWillInvokeCaptureTextFromCameraEnabled {
FlutterEngine* mockEngine = OCMPartialMock([[FlutterEngine alloc] init]);
[mockEngine createShell:@"" libraryURI:@"" initialRoute:nil];
FlutterViewController* viewController = [[FlutterViewController alloc] initWithEngine:mockEngine
nibName:nil
bundle:nil];
FlutterViewController* viewControllerMock = OCMPartialMock(viewController);
NSNotification* fakeNotification = [NSNotification notificationWithName:@"" object:nil];
[viewControllerMock onUserSettingsChanged:fakeNotification];
OCMVerify([viewControllerMock captureTextFromCameraEnabled]);
}

- (void)testkeyboardWillChangeFrameWillStartKeyboardAnimation {
FlutterEngine* mockEngine = OCMPartialMock([[FlutterEngine alloc] init]);
[mockEngine createShell:@"" libraryURI:@"" initialRoute:nil];
Expand Down