From 8e847efc29f674bf86a5905fa31c9537c1a61d74 Mon Sep 17 00:00:00 2001 From: mportune-bw Date: Mon, 13 Jan 2020 11:05:57 -0500 Subject: [PATCH 1/3] Trigger overlay prompt when focusing on username field --- .../Accessibility/AccessibilityHelpers.cs | 23 ++++++++ .../Accessibility/AccessibilityService.cs | 52 ++++++++++--------- 2 files changed, 50 insertions(+), 25 deletions(-) diff --git a/src/Android/Accessibility/AccessibilityHelpers.cs b/src/Android/Accessibility/AccessibilityHelpers.cs index 35687bf454d..8c526523801 100644 --- a/src/Android/Accessibility/AccessibilityHelpers.cs +++ b/src/Android/Accessibility/AccessibilityHelpers.cs @@ -268,6 +268,29 @@ public static AccessibilityNodeInfo GetUsernameEditText(IEnumerable !n.Password).LastOrDefault(); } + public static bool IsUsernameEditText(AccessibilityNodeInfo root, AccessibilityEvent e) + { + var passwordNodes = AccessibilityHelpers.GetWindowNodes(root, e, n => n.Password, false); + if(passwordNodes.Count > 0) + { + var allEditTexts = GetWindowNodes(root, e, n => EditText(n), false); + var usernameEditText = GetUsernameEditText(allEditTexts); + var isUsernameEditText = IsSameNode(usernameEditText, e.Source); + allEditTexts.Dispose(); + usernameEditText = null; + return isUsernameEditText; + } + return false; + } + + public static bool IsSameNode(AccessibilityNodeInfo info1, AccessibilityNodeInfo info2) + { + if(info1 != null && info2 != null) { + return info1.Equals(info2) || info1.GetHashCode() == info2.GetHashCode(); + } + return false; + } + public static bool OverlayPermitted() { if(Build.VERSION.SdkInt >= BuildVersionCodes.M) diff --git a/src/Android/Accessibility/AccessibilityService.cs b/src/Android/Accessibility/AccessibilityService.cs index d991c497edf..eed2d6b9d94 100644 --- a/src/Android/Accessibility/AccessibilityService.cs +++ b/src/Android/Accessibility/AccessibilityService.cs @@ -73,12 +73,12 @@ public override void OnAccessibilityEvent(AccessibilityEvent e) { break; } - if(e.Source == null || !e.Source.Password) + if(e.PackageName == BitwardenPackage) { CancelOverlayPrompt(); break; } - if(e.PackageName == BitwardenPackage) + if(e.Source == null || (!e.Source.Password && !AccessibilityHelpers.IsUsernameEditText(root, e))) { CancelOverlayPrompt(); break; @@ -94,7 +94,7 @@ public override void OnAccessibilityEvent(AccessibilityEvent e) break; case EventTypes.WindowContentChanged: case EventTypes.WindowStateChanged: - if(e.Source == null || e.Source.Password) + if(e.Source == null || e.Source.Password || AccessibilityHelpers.IsUsernameEditText(root, e)) { break; } @@ -127,6 +127,11 @@ public override void OnAccessibilityEvent(AccessibilityEvent e) { CancelOverlayPrompt(); } + + if(_overlayView != null) + { + OverlayPromptToAutofill(root, e); + } break; default: break; @@ -229,41 +234,38 @@ private void OverlayPromptToAutofill(AccessibilityNodeInfo root, AccessibilityEv layoutParams.X = anchorPosition.X; layoutParams.Y = anchorPosition.Y; - var intent = new Intent(this, typeof(AccessibilityActivity)); - intent.PutExtra("uri", uri); - intent.SetFlags(ActivityFlags.NewTask | ActivityFlags.SingleTop | ActivityFlags.ClearTop); - if(_windowManager == null) { _windowManager = GetSystemService(WindowService).JavaCast(); } - var updateView = false; - if(_overlayView != null) + if(_overlayView == null) { - updateView = true; - } + var intent = new Intent(this, typeof(AccessibilityActivity)); + intent.PutExtra("uri", uri); + intent.SetFlags(ActivityFlags.NewTask | ActivityFlags.SingleTop | ActivityFlags.ClearTop); - _overlayView = AccessibilityHelpers.GetOverlayView(this); - _overlayView.Click += (sender, eventArgs) => - { - CancelOverlayPrompt(); - StartActivity(intent); - }; + _overlayView = AccessibilityHelpers.GetOverlayView(this); + _overlayView.Click += (sender, eventArgs) => + { + CancelOverlayPrompt(); + StartActivity(intent); + }; - _lastNotificationUri = uri; + _lastNotificationUri = uri; - if(updateView) - { - _windowManager.UpdateViewLayout(_overlayView, layoutParams); + _windowManager.AddView(_overlayView, layoutParams); + + System.Diagnostics.Debug.WriteLine(">>> Accessibility Overlay View Added at X:{0} Y:{1}", + layoutParams.X, layoutParams.Y); } else { - _windowManager.AddView(_overlayView, layoutParams); - } + _windowManager.UpdateViewLayout(_overlayView, layoutParams); - System.Diagnostics.Debug.WriteLine(">>> Accessibility Overlay View {0} X:{1} Y:{2}", - updateView ? "Updated to" : "Added at", layoutParams.X, layoutParams.Y); + System.Diagnostics.Debug.WriteLine(">>> Accessibility Overlay View Updated to X:{0} Y:{1}", + layoutParams.X, layoutParams.Y); + } } private bool SkipPackage(string eventPackageName) From 23258972c5977af003827d2812f448e97f7f6a61 Mon Sep 17 00:00:00 2001 From: mportune-bw Date: Mon, 13 Jan 2020 15:50:21 -0500 Subject: [PATCH 2/3] Adjust accessibility overlay position in response to scroll events --- .../Accessibility/AccessibilityHelpers.cs | 53 ++++++++-- .../Accessibility/AccessibilityService.cs | 97 ++++++++++++------- .../Resources/xml/accessibilityservice.xml | 2 +- 3 files changed, 110 insertions(+), 42 deletions(-) diff --git a/src/Android/Accessibility/AccessibilityHelpers.cs b/src/Android/Accessibility/AccessibilityHelpers.cs index 8c526523801..87c2e0fe34f 100644 --- a/src/Android/Accessibility/AccessibilityHelpers.cs +++ b/src/Android/Accessibility/AccessibilityHelpers.cs @@ -317,20 +317,59 @@ public static LinearLayout GetOverlayView(Context context) return view; } - public static Point GetOverlayAnchorPosition(AccessibilityNodeInfo root, AccessibilityEvent e) + public static WindowManagerLayoutParams GetOverlayLayoutParams() + { + WindowManagerTypes windowManagerType; + if(Build.VERSION.SdkInt >= BuildVersionCodes.O) + { + windowManagerType = WindowManagerTypes.ApplicationOverlay; + } + else + { + windowManagerType = WindowManagerTypes.Phone; + } + + var layoutParams = new WindowManagerLayoutParams( + ViewGroup.LayoutParams.WrapContent, + ViewGroup.LayoutParams.WrapContent, + windowManagerType, + WindowManagerFlags.NotFocusable | WindowManagerFlags.NotTouchModal, + Format.Transparent); + layoutParams.Gravity = GravityFlags.Bottom | GravityFlags.Left; + + return layoutParams; + } + + public static Point GetOverlayAnchorPosition(AccessibilityNodeInfo root, AccessibilityNodeInfo anchorView) { var rootRect = new Rect(); root.GetBoundsInScreen(rootRect); var rootRectHeight = rootRect.Height(); - var eSrcRect = new Rect(); - e.Source.GetBoundsInScreen(eSrcRect); - var eSrcRectLeft = eSrcRect.Left; - var eSrcRectTop = eSrcRect.Top; + var anchorViewRect = new Rect(); + anchorView.GetBoundsInScreen(anchorViewRect); + var anchorViewRectLeft = anchorViewRect.Left; + var anchorViewRectTop = anchorViewRect.Top; var navBarHeight = GetNavigationBarHeight(); - var calculatedTop = rootRectHeight - eSrcRectTop - navBarHeight; - return new Point(eSrcRectLeft, calculatedTop); + var calculatedTop = rootRectHeight - anchorViewRectTop - navBarHeight; + return new Point(anchorViewRectLeft, calculatedTop); + } + + public static Point GetOverlayAnchorPosition(int nodeHash, AccessibilityNodeInfo root, AccessibilityEvent e) + { + Point point = null; + var allEditTexts = GetWindowNodes(root, e, n => EditText(n), false); + foreach(AccessibilityNodeInfo node in allEditTexts) + { + if(node.GetHashCode() == nodeHash) + { + point = GetOverlayAnchorPosition(root, node); + break; + } + } + allEditTexts.Dispose(); + return point; } private static int GetStatusBarHeight() diff --git a/src/Android/Accessibility/AccessibilityService.cs b/src/Android/Accessibility/AccessibilityService.cs index eed2d6b9d94..c79c7cec25a 100644 --- a/src/Android/Accessibility/AccessibilityService.cs +++ b/src/Android/Accessibility/AccessibilityService.cs @@ -35,6 +35,8 @@ public class AccessibilityService : Android.AccessibilityServices.AccessibilityS private IWindowManager _windowManager = null; private LinearLayout _overlayView = null; + private int _anchorViewHash = 0; + private int _lastAnchorX, _lastAnchorY = 0; public override void OnAccessibilityEvent(AccessibilityEvent e) { @@ -62,39 +64,51 @@ public override void OnAccessibilityEvent(AccessibilityEvent e) return; } + var isUsernameEditText = AccessibilityHelpers.IsUsernameEditText(root, e); + var isPasswordEditText = e.Source != null && e.Source.Password; + // AccessibilityHelpers.PrintTestData(root, e); switch(e.EventType) { case EventTypes.ViewFocused: case EventTypes.ViewClicked: + case EventTypes.ViewScrolled: var isKnownBroswer = AccessibilityHelpers.SupportedBrowsers.ContainsKey(root.PackageName); if(e.EventType == EventTypes.ViewClicked && isKnownBroswer) { break; } - if(e.PackageName == BitwardenPackage) + if(e.Source == null || e.PackageName == BitwardenPackage) { CancelOverlayPrompt(); break; } - if(e.Source == null || (!e.Source.Password && !AccessibilityHelpers.IsUsernameEditText(root, e))) + if (e.EventType == EventTypes.ViewScrolled) { - CancelOverlayPrompt(); + AdjustOverlayForScroll(root, e); break; } - if(ScanAndAutofill(root, e)) - { - CancelOverlayPrompt(); - } else { - OverlayPromptToAutofill(root, e); + if(!isUsernameEditText && !isPasswordEditText) + { + CancelOverlayPrompt(); + break; + } + if(ScanAndAutofill(root, e)) + { + CancelOverlayPrompt(); + } + else + { + OverlayPromptToAutofill(root, e); + } } break; case EventTypes.WindowContentChanged: case EventTypes.WindowStateChanged: - if(e.Source == null || e.Source.Password || AccessibilityHelpers.IsUsernameEditText(root, e)) + if(e.Source == null || isUsernameEditText || isPasswordEditText) { break; } @@ -127,11 +141,6 @@ public override void OnAccessibilityEvent(AccessibilityEvent e) { CancelOverlayPrompt(); } - - if(_overlayView != null) - { - OverlayPromptToAutofill(root, e); - } break; default: break; @@ -193,7 +202,10 @@ private void CancelOverlayPrompt() System.Diagnostics.Debug.WriteLine(">>> Accessibility Overlay View Removed"); _overlayView = null; + _anchorViewHash = 0; _lastNotificationUri = null; + _lastAnchorX = 0; + _lastAnchorY = 0; } private void OverlayPromptToAutofill(AccessibilityNodeInfo root, AccessibilityEvent e) @@ -211,26 +223,8 @@ private void OverlayPromptToAutofill(AccessibilityNodeInfo root, AccessibilityEv return; } - WindowManagerTypes windowManagerType; - if(Build.VERSION.SdkInt >= BuildVersionCodes.O) - { - windowManagerType = WindowManagerTypes.ApplicationOverlay; - } - else - { - windowManagerType = WindowManagerTypes.Phone; - } - - var layoutParams = new WindowManagerLayoutParams( - ViewGroup.LayoutParams.WrapContent, - ViewGroup.LayoutParams.WrapContent, - windowManagerType, - WindowManagerFlags.NotFocusable | WindowManagerFlags.NotTouchModal, - Format.Transparent); - - var anchorPosition = AccessibilityHelpers.GetOverlayAnchorPosition(root, e); - - layoutParams.Gravity = GravityFlags.Bottom | GravityFlags.Left; + var layoutParams = AccessibilityHelpers.GetOverlayLayoutParams(); + var anchorPosition = AccessibilityHelpers.GetOverlayAnchorPosition(root, e.Source); layoutParams.X = anchorPosition.X; layoutParams.Y = anchorPosition.Y; @@ -266,6 +260,41 @@ private void OverlayPromptToAutofill(AccessibilityNodeInfo root, AccessibilityEv System.Diagnostics.Debug.WriteLine(">>> Accessibility Overlay View Updated to X:{0} Y:{1}", layoutParams.X, layoutParams.Y); } + + _anchorViewHash = e.Source.GetHashCode(); + _lastAnchorX = anchorPosition.X; + _lastAnchorY = anchorPosition.Y; + } + + private void AdjustOverlayForScroll(AccessibilityNodeInfo root, AccessibilityEvent e) + { + if(_overlayView == null || _anchorViewHash <= 0) + { + return; + } + + Point anchorPosition = AccessibilityHelpers.GetOverlayAnchorPosition(_anchorViewHash, root, e); + if(anchorPosition == null) + { + return; + } + + if(anchorPosition.X == _lastAnchorX && anchorPosition.Y == _lastAnchorY) + { + return; + } + + var layoutParams = AccessibilityHelpers.GetOverlayLayoutParams(); + layoutParams.X = anchorPosition.X; + layoutParams.Y = anchorPosition.Y; + + _windowManager.UpdateViewLayout(_overlayView, layoutParams); + + _lastAnchorX = anchorPosition.X; + _lastAnchorY = anchorPosition.Y; + + System.Diagnostics.Debug.WriteLine(">>> Accessibility Overlay View Updated to X:{0} Y:{1}", + layoutParams.X, layoutParams.Y); } private bool SkipPackage(string eventPackageName) diff --git a/src/Android/Resources/xml/accessibilityservice.xml b/src/Android/Resources/xml/accessibilityservice.xml index 5b1e700198f..674759ad6cb 100644 --- a/src/Android/Resources/xml/accessibilityservice.xml +++ b/src/Android/Resources/xml/accessibilityservice.xml @@ -2,7 +2,7 @@ Date: Mon, 13 Jan 2020 16:46:17 -0500 Subject: [PATCH 3/3] Get username EditText with a single pass of the node tree, plus additional cleanup --- .../Accessibility/AccessibilityHelpers.cs | 30 ++++++++++++------- .../Accessibility/AccessibilityService.cs | 15 +++++----- 2 files changed, 28 insertions(+), 17 deletions(-) diff --git a/src/Android/Accessibility/AccessibilityHelpers.cs b/src/Android/Accessibility/AccessibilityHelpers.cs index 87c2e0fe34f..92041674fbe 100644 --- a/src/Android/Accessibility/AccessibilityHelpers.cs +++ b/src/Android/Accessibility/AccessibilityHelpers.cs @@ -257,24 +257,33 @@ public static void GetNodesAndFill(AccessibilityNodeInfo root, AccessibilityEven IEnumerable passwordNodes) { var allEditTexts = GetWindowNodes(root, e, n => EditText(n), false); - var usernameEditText = GetUsernameEditText(allEditTexts); + var usernameEditText = GetUsernameEditTextIfPasswordExists(allEditTexts); FillCredentials(usernameEditText, passwordNodes); allEditTexts.Dispose(); usernameEditText = null; } - public static AccessibilityNodeInfo GetUsernameEditText(IEnumerable allEditTexts) + public static AccessibilityNodeInfo GetUsernameEditTextIfPasswordExists( + IEnumerable allEditTexts) { - return allEditTexts.TakeWhile(n => !n.Password).LastOrDefault(); + AccessibilityNodeInfo previousEditText = null; + foreach(var editText in allEditTexts) + { + if(editText.Password) + { + return previousEditText; + } + previousEditText = editText; + } + return null; } public static bool IsUsernameEditText(AccessibilityNodeInfo root, AccessibilityEvent e) { - var passwordNodes = AccessibilityHelpers.GetWindowNodes(root, e, n => n.Password, false); - if(passwordNodes.Count > 0) - { - var allEditTexts = GetWindowNodes(root, e, n => EditText(n), false); - var usernameEditText = GetUsernameEditText(allEditTexts); + var allEditTexts = GetWindowNodes(root, e, n => EditText(n), false); + var usernameEditText = GetUsernameEditTextIfPasswordExists(allEditTexts); + if(usernameEditText != null) + { var isUsernameEditText = IsSameNode(usernameEditText, e.Source); allEditTexts.Dispose(); usernameEditText = null; @@ -285,7 +294,8 @@ public static bool IsUsernameEditText(AccessibilityNodeInfo root, AccessibilityE public static bool IsSameNode(AccessibilityNodeInfo info1, AccessibilityNodeInfo info2) { - if(info1 != null && info2 != null) { + if(info1 != null && info2 != null) + { return info1.Equals(info2) || info1.GetHashCode() == info2.GetHashCode(); } return false; @@ -360,7 +370,7 @@ public static Point GetOverlayAnchorPosition(int nodeHash, AccessibilityNodeInfo { Point point = null; var allEditTexts = GetWindowNodes(root, e, n => EditText(n), false); - foreach(AccessibilityNodeInfo node in allEditTexts) + foreach(var node in allEditTexts) { if(node.GetHashCode() == nodeHash) { diff --git a/src/Android/Accessibility/AccessibilityService.cs b/src/Android/Accessibility/AccessibilityService.cs index c79c7cec25a..c9f591ad647 100644 --- a/src/Android/Accessibility/AccessibilityService.cs +++ b/src/Android/Accessibility/AccessibilityService.cs @@ -64,9 +64,6 @@ public override void OnAccessibilityEvent(AccessibilityEvent e) return; } - var isUsernameEditText = AccessibilityHelpers.IsUsernameEditText(root, e); - var isPasswordEditText = e.Source != null && e.Source.Password; - // AccessibilityHelpers.PrintTestData(root, e); switch(e.EventType) @@ -84,14 +81,16 @@ public override void OnAccessibilityEvent(AccessibilityEvent e) CancelOverlayPrompt(); break; } - if (e.EventType == EventTypes.ViewScrolled) + if(e.EventType == EventTypes.ViewScrolled) { AdjustOverlayForScroll(root, e); break; } else { - if(!isUsernameEditText && !isPasswordEditText) + var isUsernameEditText1 = AccessibilityHelpers.IsUsernameEditText(root, e); + var isPasswordEditText1 = e.Source?.Password ?? false; + if(!isUsernameEditText1 && !isPasswordEditText1) { CancelOverlayPrompt(); break; @@ -108,7 +107,9 @@ public override void OnAccessibilityEvent(AccessibilityEvent e) break; case EventTypes.WindowContentChanged: case EventTypes.WindowStateChanged: - if(e.Source == null || isUsernameEditText || isPasswordEditText) + var isUsernameEditText2 = AccessibilityHelpers.IsUsernameEditText(root, e); + var isPasswordEditText2 = e.Source?.Password ?? false; + if(e.Source == null || isUsernameEditText2 || isPasswordEditText2) { break; } @@ -273,7 +274,7 @@ private void AdjustOverlayForScroll(AccessibilityNodeInfo root, AccessibilityEve return; } - Point anchorPosition = AccessibilityHelpers.GetOverlayAnchorPosition(_anchorViewHash, root, e); + var anchorPosition = AccessibilityHelpers.GetOverlayAnchorPosition(_anchorViewHash, root, e); if(anchorPosition == null) { return;