Skip to content

Accessibility overlay support for username field and scroll tracking #700

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
kspearrin merged 3 commits into from
Jan 13, 2020
Merged

Accessibility overlay support for username field and scroll tracking #700

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
8e847ef
Trigger overlay prompt when focusing on username field
mpbw2 Jan 13, 2020
2325897
Adjust accessibility overlay position in response to scroll events
mpbw2 Jan 13, 2020
7414e92
Get username EditText with a single pass of the node tree, plus addit…
mpbw2 Jan 13, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
92 changes: 82 additions & 10 deletions src/Android/Accessibility/AccessibilityHelpers.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -257,15 +257,48 @@ public static void GetNodesAndFill(AccessibilityNodeInfo root, AccessibilityEven
IEnumerable<AccessibilityNodeInfo> passwordNodes)
{
var allEditTexts = GetWindowNodes(root, e, n => EditText(n), false);
var usernameEditText = GetUsernameEditText(allEditTexts);
var usernameEditText = GetUsernameEditTextIfPasswordExists(allEditTexts);
FillCredentials(usernameEditText, passwordNodes);
allEditTexts.Dispose();
usernameEditText = null;
}

public static AccessibilityNodeInfo GetUsernameEditText(IEnumerable<AccessibilityNodeInfo> allEditTexts)
public static AccessibilityNodeInfo GetUsernameEditTextIfPasswordExists(
IEnumerable<AccessibilityNodeInfo> allEditTexts)
{
return allEditTexts.TakeWhile(n => !n.Password).LastOrDefault();
AccessibilityNodeInfo previousEditText = null;
foreach(var editText in allEditTexts)
{
if(editText.Password)
{
return previousEditText;
}
previousEditText = editText;
}
return null;
}

public static bool IsUsernameEditText(AccessibilityNodeInfo root, AccessibilityEvent e)
{
var allEditTexts = GetWindowNodes(root, e, n => EditText(n), false);
var usernameEditText = GetUsernameEditTextIfPasswordExists(allEditTexts);
if(usernameEditText != null)
{
var isUsernameEditText = IsSameNode(usernameEditText, e.Source);
allEditTexts.Dispose();
usernameEditText = null;
return isUsernameEditText;
}
return false;
}

public static bool IsSameNode(AccessibilityNodeInfo info1, AccessibilityNodeInfo info2)
{
if(info1 != null && info2 != null)
{
return info1.Equals(info2) || info1.GetHashCode() == info2.GetHashCode();
}
return false;
}

public static bool OverlayPermitted()
Expand Down Expand Up @@ -294,20 +327,59 @@ public static LinearLayout GetOverlayView(Context context)
return view;
}

public static Point GetOverlayAnchorPosition(AccessibilityNodeInfo root, AccessibilityEvent e)
public static WindowManagerLayoutParams GetOverlayLayoutParams()
{
WindowManagerTypes windowManagerType;
if(Build.VERSION.SdkInt >= BuildVersionCodes.O)
{
windowManagerType = WindowManagerTypes.ApplicationOverlay;
}
else
{
windowManagerType = WindowManagerTypes.Phone;
}

var layoutParams = new WindowManagerLayoutParams(
ViewGroup.LayoutParams.WrapContent,
ViewGroup.LayoutParams.WrapContent,
windowManagerType,
WindowManagerFlags.NotFocusable | WindowManagerFlags.NotTouchModal,
Format.Transparent);
layoutParams.Gravity = GravityFlags.Bottom | GravityFlags.Left;

return layoutParams;
}

public static Point GetOverlayAnchorPosition(AccessibilityNodeInfo root, AccessibilityNodeInfo anchorView)
{
var rootRect = new Rect();
root.GetBoundsInScreen(rootRect);
var rootRectHeight = rootRect.Height();

var eSrcRect = new Rect();
e.Source.GetBoundsInScreen(eSrcRect);
var eSrcRectLeft = eSrcRect.Left;
var eSrcRectTop = eSrcRect.Top;
var anchorViewRect = new Rect();
anchorView.GetBoundsInScreen(anchorViewRect);
var anchorViewRectLeft = anchorViewRect.Left;
var anchorViewRectTop = anchorViewRect.Top;

var navBarHeight = GetNavigationBarHeight();
var calculatedTop = rootRectHeight - eSrcRectTop - navBarHeight;
return new Point(eSrcRectLeft, calculatedTop);
var calculatedTop = rootRectHeight - anchorViewRectTop - navBarHeight;
return new Point(anchorViewRectLeft, calculatedTop);
}

public static Point GetOverlayAnchorPosition(int nodeHash, AccessibilityNodeInfo root, AccessibilityEvent e)
{
Point point = null;
var allEditTexts = GetWindowNodes(root, e, n => EditText(n), false);
foreach(var node in allEditTexts)
{
if(node.GetHashCode() == nodeHash)
{
point = GetOverlayAnchorPosition(root, node);
break;
}
}
allEditTexts.Dispose();
return point;
}

private static int GetStatusBarHeight()
Expand Down
130 changes: 81 additions & 49 deletions src/Android/Accessibility/AccessibilityService.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,8 @@ public class AccessibilityService : Android.AccessibilityServices.AccessibilityS

private IWindowManager _windowManager = null;
private LinearLayout _overlayView = null;
private int _anchorViewHash = 0;
private int _lastAnchorX, _lastAnchorY = 0;

public override void OnAccessibilityEvent(AccessibilityEvent e)
{
Expand Down Expand Up @@ -68,33 +70,46 @@ public override void OnAccessibilityEvent(AccessibilityEvent e)
{
case EventTypes.ViewFocused:
case EventTypes.ViewClicked:
case EventTypes.ViewScrolled:
var isKnownBroswer = AccessibilityHelpers.SupportedBrowsers.ContainsKey(root.PackageName);
if(e.EventType == EventTypes.ViewClicked && isKnownBroswer)
{
break;
}
if(e.Source == null || !e.Source.Password)
if(e.Source == null || e.PackageName == BitwardenPackage)
{
CancelOverlayPrompt();
break;
}
if(e.PackageName == BitwardenPackage)
if(e.EventType == EventTypes.ViewScrolled)
{
CancelOverlayPrompt();
AdjustOverlayForScroll(root, e);
break;
}
if(ScanAndAutofill(root, e))
{
CancelOverlayPrompt();
}
else
{
OverlayPromptToAutofill(root, e);
var isUsernameEditText1 = AccessibilityHelpers.IsUsernameEditText(root, e);
var isPasswordEditText1 = e.Source?.Password ?? false;
if(!isUsernameEditText1 && !isPasswordEditText1)
{
CancelOverlayPrompt();
break;
}
if(ScanAndAutofill(root, e))
{
CancelOverlayPrompt();
}
else
{
OverlayPromptToAutofill(root, e);
}
}
break;
case EventTypes.WindowContentChanged:
case EventTypes.WindowStateChanged:
if(e.Source == null || e.Source.Password)
var isUsernameEditText2 = AccessibilityHelpers.IsUsernameEditText(root, e);
var isPasswordEditText2 = e.Source?.Password ?? false;
if(e.Source == null || isUsernameEditText2 || isPasswordEditText2)
{
break;
}
Expand Down Expand Up @@ -188,7 +203,10 @@ private void CancelOverlayPrompt()
System.Diagnostics.Debug.WriteLine(">>> Accessibility Overlay View Removed");

_overlayView = null;
_anchorViewHash = 0;
_lastNotificationUri = null;
_lastAnchorX = 0;
_lastAnchorY = 0;
}

private void OverlayPromptToAutofill(AccessibilityNodeInfo root, AccessibilityEvent e)
Expand All @@ -206,64 +224,78 @@ private void OverlayPromptToAutofill(AccessibilityNodeInfo root, AccessibilityEv
return;
}

WindowManagerTypes windowManagerType;
if(Build.VERSION.SdkInt >= BuildVersionCodes.O)
{
windowManagerType = WindowManagerTypes.ApplicationOverlay;
}
else
{
windowManagerType = WindowManagerTypes.Phone;
}

var layoutParams = new WindowManagerLayoutParams(
ViewGroup.LayoutParams.WrapContent,
ViewGroup.LayoutParams.WrapContent,
windowManagerType,
WindowManagerFlags.NotFocusable | WindowManagerFlags.NotTouchModal,
Format.Transparent);

var anchorPosition = AccessibilityHelpers.GetOverlayAnchorPosition(root, e);

layoutParams.Gravity = GravityFlags.Bottom | GravityFlags.Left;
var layoutParams = AccessibilityHelpers.GetOverlayLayoutParams();
var anchorPosition = AccessibilityHelpers.GetOverlayAnchorPosition(root, e.Source);
layoutParams.X = anchorPosition.X;
layoutParams.Y = anchorPosition.Y;

var intent = new Intent(this, typeof(AccessibilityActivity));
intent.PutExtra("uri", uri);
intent.SetFlags(ActivityFlags.NewTask | ActivityFlags.SingleTop | ActivityFlags.ClearTop);

if(_windowManager == null)
{
_windowManager = GetSystemService(WindowService).JavaCast<IWindowManager>();
}

var updateView = false;
if(_overlayView != null)
if(_overlayView == null)
{
updateView = true;
}
var intent = new Intent(this, typeof(AccessibilityActivity));
intent.PutExtra("uri", uri);
intent.SetFlags(ActivityFlags.NewTask | ActivityFlags.SingleTop | ActivityFlags.ClearTop);

_overlayView = AccessibilityHelpers.GetOverlayView(this);
_overlayView.Click += (sender, eventArgs) =>
{
CancelOverlayPrompt();
StartActivity(intent);
};
_overlayView = AccessibilityHelpers.GetOverlayView(this);
_overlayView.Click += (sender, eventArgs) =>
{
CancelOverlayPrompt();
StartActivity(intent);
};

_lastNotificationUri = uri;

_lastNotificationUri = uri;
_windowManager.AddView(_overlayView, layoutParams);

if(updateView)
System.Diagnostics.Debug.WriteLine(">>> Accessibility Overlay View Added at X:{0} Y:{1}",
layoutParams.X, layoutParams.Y);
}
else
{
_windowManager.UpdateViewLayout(_overlayView, layoutParams);

System.Diagnostics.Debug.WriteLine(">>> Accessibility Overlay View Updated to X:{0} Y:{1}",
layoutParams.X, layoutParams.Y);
}
else

_anchorViewHash = e.Source.GetHashCode();
_lastAnchorX = anchorPosition.X;
_lastAnchorY = anchorPosition.Y;
}

private void AdjustOverlayForScroll(AccessibilityNodeInfo root, AccessibilityEvent e)
{
if(_overlayView == null || _anchorViewHash <= 0)
{
_windowManager.AddView(_overlayView, layoutParams);
return;
}

System.Diagnostics.Debug.WriteLine(">>> Accessibility Overlay View {0} X:{1} Y:{2}",
updateView ? "Updated to" : "Added at", layoutParams.X, layoutParams.Y);
var anchorPosition = AccessibilityHelpers.GetOverlayAnchorPosition(_anchorViewHash, root, e);
if(anchorPosition == null)
{
return;
}

if(anchorPosition.X == _lastAnchorX && anchorPosition.Y == _lastAnchorY)
{
return;
}

var layoutParams = AccessibilityHelpers.GetOverlayLayoutParams();
layoutParams.X = anchorPosition.X;
layoutParams.Y = anchorPosition.Y;

_windowManager.UpdateViewLayout(_overlayView, layoutParams);

_lastAnchorX = anchorPosition.X;
_lastAnchorY = anchorPosition.Y;

System.Diagnostics.Debug.WriteLine(">>> Accessibility Overlay View Updated to X:{0} Y:{1}",
layoutParams.X, layoutParams.Y);
}

private bool SkipPackage(string eventPackageName)
Expand Down
2 changes: 1 addition & 1 deletion src/Android/Resources/xml/accessibilityservice.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
<accessibility-service xmlns:android="http://schemas.android.com/apk/res/android"
android:summary="@string/AutoFillServiceSummary"
android:description="@string/AutoFillServiceDescription"
android:accessibilityEventTypes="typeWindowStateChanged|typeWindowContentChanged|typeViewFocused|typeViewClicked"
android:accessibilityEventTypes="typeWindowStateChanged|typeWindowContentChanged|typeViewFocused|typeViewClicked|typeViewScrolled"
android:accessibilityFeedbackType="feedbackGeneric"
android:accessibilityFlags="flagReportViewIds"
android:notificationTimeout="100"
Expand Down