Skip to content
Permalink
Browse files

Fix window point conversion functions not exposing failure (#9269)

* Remove legacy Windows xp support from windowUtils

* displayModel, deal with new behavior for logicalToPhysical and physicalToLogical

* winUser, update screenToClient and clientToScreen

* Update locationHelper

* Update edit

* Fix bad copy paste in debug warning

* Fix the use of ScreenToClient in MSHTML

* ScreenToClient error catching for Window and Edit

* Review actions

* Fix unclear or broken comments

* Use locationHelper in mshtml

* displayModel, compensate for left coordinates greater than right

* Linting issues

* Uniformize string formatting

* Revert "displayModel, compensate for left coordinates greater than right"

This reverts commit 0324936.
  • Loading branch information
leonardder authored and michaelDCurran committed Jan 13, 2020
1 parent b9c4534 commit 669ae5ac95dc8e7047131260d92f0792f4e35197
@@ -27,7 +27,7 @@
from .. import InvalidNVDAObject
from ..window import Window
from NVDAObjects.UIA import UIA, UIATextInfo
from locationHelper import RectLTRB
from locationHelper import RectLTRB, Point
from typing import Dict

IID_IHTMLElement=comtypes.GUID('{3050F1FF-98B5-11CF-BB82-00AA00BDCE0B}')
@@ -487,14 +487,20 @@ def kwargsFromSuper(cls,kwargs,relation=None):

elif isinstance(relation,tuple):
windowHandle=kwargs.get('windowHandle')
p=ctypes.wintypes.POINT(x=relation[0],y=relation[1])
ctypes.windll.user32.ScreenToClient(windowHandle,ctypes.byref(p))
if not windowHandle:
log.debugWarning("Error converting point to client coordinates, no window handle")
return False
try:
point = Point(*relation).toClient(windowHandle)
except WindowsError:
log.debugWarning("Error converting point to client coordinates", exc_info=True)
return False
# #3494: MSHTML's internal coordinates are always at a hardcoded DPI (usually 96) no matter the system DPI or zoom level.
xFactor,yFactor=getZoomFactorsFromHTMLDocument(HTMLNode.document)
try:
HTMLNode=HTMLNode.document.elementFromPoint(p.x // xFactor, p.y // yFactor)
HTMLNode = HTMLNode.document.elementFromPoint(point.x // xFactor, point.y // yFactor)
except:
HTMLNode=None
HTMLNode = None
if not HTMLNode:
log.debugWarning("Error getting HTMLNode with elementFromPoint")
return False
@@ -202,12 +202,18 @@ def _get_displayText(self):

def redraw(self):
"""Redraw the display for this object.
@raise WindowsError: If redrawing fails.
"""
left, top, width, height = self.location
left, top = winUser.ScreenToClient(self.windowHandle, left, top)
winUser.RedrawWindow(self.windowHandle,
winUser.RECT(left, top, left + width, top + height), None,
winUser.RDW_INVALIDATE | winUser.RDW_UPDATENOW)
# Conversion to client coordinates may fail if the window handle of this object is incorrect.
# This will most likely be caused by a died window.
location = self.location.toClient(self.windowHandle)
if not winUser.RedrawWindow(
self.windowHandle,
location.toRECT(),
None,
winUser.RDW_INVALIDATE | winUser.RDW_UPDATENOW
):
raise ctypes.WinError()

def _get_windowText(self):
textLength=watchdog.cancellableSendMessage(self.windowHandle,winUser.WM_GETTEXTLENGTH,0,0)
@@ -404,6 +410,8 @@ class DisplayModelLiveText(LiveText, Window):

def startMonitoring(self):
# Force the window to be redrawn, as our display model might be out of date.
# Do not catch exceptions caused by redraw, as when redrawing fails,
# it is most likely that the window died, and we don't want to monitor in that case.
self.redraw()
displayModel.requestTextChangeNotifications(self, True)
super(DisplayModelLiveText, self).startMonitoring()
@@ -180,11 +180,18 @@ def _getPointFromOffset(self,offset):
if point.x <0 or point.y <0:
raise LookupError("Point with client coordinates x=%d, y=%d not within client area of object" %
(point.x, point.y))
return point.toScreen(self.obj.windowHandle)

try:
return point.toScreen(self.obj.windowHandle)
except WindowsError as e:
raise LookupError(
f"Couldn't convert point at offset {offset} to screen coordinates: {e.strerror}"
)

def _getOffsetFromPoint(self,x,y):
x, y = winUser.ScreenToClient(self.obj.windowHandle, x, y)
try:
x, y = winUser.ScreenToClient(self.obj.windowHandle, x, y)
except WindowsError as e:
raise LookupError(f"Couldn't convert point ({x},{y}) to client coordinates: {e.strerror}")
if self.obj.editAPIVersion>=1:
processHandle=self.obj.processHandle
internalP=winKernel.virtualAllocEx(processHandle,None,ctypes.sizeof(PointLStruct),winKernel.MEM_COMMIT,winKernel.PAGE_READWRITE)
@@ -21,7 +21,7 @@
import watchdog
from logHandler import log
import windowUtils
from locationHelper import RectLTRB, RectLTWH
from locationHelper import RectLTRB, RectLTWH, Point
import textUtils
from typing import Union, List, Tuple

@@ -300,23 +300,31 @@ def _get__storyFieldsAndRects(self) -> Tuple[
List[int]
]:
# All returned coordinates are logical coordinates.
if self._location:
left, top, right, bottom = self._location
else:
try:
left, top, width, height = self.obj.location
except TypeError:
# No location; nothing we can do.
return [], [], [], []
right = left + width
bottom = top + height
location = self._location if self._location else self.obj.location
if location is None or not any(location):
# No location; nothing we can do.
return [], [], [], []
bindingHandle=self.obj.appModule.helperLocalBindingHandle
if not bindingHandle:
log.debugWarning("AppModule does not have a binding handle")
return [], [], [], []
left,top=windowUtils.physicalToLogicalPoint(self.obj.windowHandle,left,top)
right,bottom=windowUtils.physicalToLogicalPoint(self.obj.windowHandle,right,bottom)
text,rects=getWindowTextInRect(bindingHandle, self.obj.windowHandle, left, top, right, bottom, self.minHorizontalWhitespace, self.minVerticalWhitespace,self.stripOuterWhitespace,self.includeDescendantWindows)
try:
location = location.toLogical(self.obj.windowHandle)
except RuntimeError:
log.exception()
return [], [], [], []
text, rects = getWindowTextInRect(
bindingHandle,
self.obj.windowHandle,
location.left,
location.top,
location.right,
location.bottom,
self.minHorizontalWhitespace,
self.minVerticalWhitespace,
self.stripOuterWhitespace,
self.includeDescendantWindows
)
if not text:
return [], [], [], []
text="<control>%s</control>"%text
@@ -444,15 +452,29 @@ def _normalizeFormatField(self,field):

def _getOffsetFromPoint(self, x, y):
# Accepts physical coordinates.
x,y=windowUtils.physicalToLogicalPoint(self.obj.windowHandle,x,y)
try:
x, y = windowUtils.physicalToLogicalPoint(
self.obj.windowHandle,
x,
y
)
except RuntimeError:
raise LookupError("physicalToLogicalPoint failed")
for charOffset, (charLeft, charTop, charRight, charBottom) in enumerate(self._storyFieldsAndRects[1]):
if charLeft<=x<charRight and charTop<=y<charBottom:
return charOffset
raise LookupError

def _getClosestOffsetFromPoint(self,x,y):
# Accepts physical coordinates.
x,y=windowUtils.physicalToLogicalPoint(self.obj.windowHandle,x,y)
try:
x, y = windowUtils.physicalToLogicalPoint(
self.obj.windowHandle,
x,
y
)
except RuntimeError:
raise LookupError("physicalToLogicalPoint failed")
#Enumerate the character rectangles
a=enumerate(self._storyFieldsAndRects[1])
#Convert calculate center points for all the rectangles
@@ -470,7 +492,14 @@ def _getBoundingRectFromOffset(self, offset):
rects=self._storyFieldsAndRects[1]
if not rects or offset>=len(rects):
raise LookupError
return rects[offset].toPhysical(self.obj.windowHandle).toLTWH()
rect = rects[offset].toLTWH()
try:
rect = rect.toPhysical(self.obj.windowHandle)
except RuntimeError:
raise LookupError(
f"Couldn't convert character rectangle at offset {offset} to physical coordinates"
)
return rect

def _getNVDAObjectFromOffset(self,offset):
try:
@@ -562,7 +591,15 @@ def _get_boundingRects(self):
for lineEndOffset in lineEndOffsets:
startOffset=endOffset
endOffset=lineEndOffset
rects.append(RectLTWH.fromCollection(*self._storyFieldsAndRects[1][startOffset:endOffset]).toPhysical(self.obj.windowHandle))
lineRect = RectLTWH.fromCollection(*self._storyFieldsAndRects[1][startOffset:endOffset])
try:
lineRect = lineRect.toPhysical(self.obj.windowHandle)
except RuntimeError:
raise LookupError(
f"Couldn't convert line rectangle at offsets {startOffset} to {endOffset} "
"to physical coordinates"
)
rects.append(lineRect)
return rects

def _getFirstVisibleOffset(self):
@@ -606,7 +643,12 @@ def _findCaretOffsetFromLocation(
def _getCaretOffset(self):
caretRect = getCaretRect(self.obj)
objLocation = self.obj.location
objRect = objLocation.toLTRB().toLogical(self.obj.windowHandle)
try:
objRect = objLocation.toLTRB().toLogical(self.obj.windowHandle)
except RuntimeError:
raise RuntimeError(
"Couldn't convert object location to logical coordinates when getting caret offset"
)
caretRect = caretRect.intersection(objRect)
if not any(caretRect):
raise RuntimeError("The caret rectangle does not overlap with the window")
@@ -631,11 +673,14 @@ def _setCaretOffset(self,offset):
if offset>=len(rects):
raise RuntimeError("offset %d out of range")
rect = rects[offset]
x = rect.left
y= rect.center.y
x,y=windowUtils.logicalToPhysicalPoint(self.obj.windowHandle,x,y)
oldX,oldY=winUser.getCursorPos()
winUser.setCursorPos(x,y)
# Place the cursor at the left coordinate of the character, vertically centered.
point = Point(rect.left, rect.center.y)
try:
point = point.toPhysical(self.obj.windowHandle)
except RuntimeError:
raise RuntimeError("Conversion to physical coordinates failed when setting caret offset")
oldX, oldY = winUser.getCursorPos()
winUser.setCursorPos(*point)
mouseHandler.executeMouseEvent(winUser.MOUSEEVENTF_LEFTDOWN,0,0)
mouseHandler.executeMouseEvent(winUser.MOUSEEVENTF_LEFTUP,0,0)
winUser.setCursorPos(oldX,oldY)
@@ -254,18 +254,24 @@ def toRECT(self):
return RECT(self.left,self.top,self.right,self.bottom)

def toLogical(self, hwnd):
left,top=self.topLeft.toLogical(hwnd)
right,bottom=self.bottomRight.toLogical(hwnd)
try:
left, top = self.topLeft.toLogical(hwnd)
right, bottom = self.bottomRight.toLogical(hwnd)
except RuntimeError:
raise RuntimeError(f"Couldn't convert {self} to logical coordinates")
if isinstance(self, RectLTWH):
return RectLTWH(left,top,right-left,bottom-top)
return RectLTRB(left,top,right,bottom)
return RectLTWH(left, top, right - left, bottom - top)
return RectLTRB(left, top, right, bottom)

def toPhysical(self, hwnd):
left,top=self.topLeft.toPhysical(hwnd)
right,bottom=self.bottomRight.toPhysical(hwnd)
try:
left, top = self.topLeft.toPhysical(hwnd)
right, bottom = self.bottomRight.toPhysical(hwnd)
except RuntimeError:
raise RuntimeError(f"Couldn't convert {self} to physical coordinates")
if isinstance(self, RectLTWH):
return RectLTWH(left,top,right-left,bottom-top)
return RectLTRB(left,top,right,bottom)
return RectLTWH(left, top, right - left, bottom - top)
return RectLTRB(left, top, right, bottom)

def toClient(self, hwnd):
left, top =self.topLeft.toClient(hwnd)
@@ -274,7 +280,7 @@ def toClient(self, hwnd):
return RectLTRB(left, top, left+self.width, top+self.height)

def toScreen(self, hwnd):
left,top=self.topLeft.toScreen(hwnd)
left, top = self.topLeft.toScreen(hwnd)
if isinstance(self, RectLTWH):
return RectLTWH(left, top, self.width, self.height)
return RectLTRB(left, top, left+self.width, top+self.height)
@@ -609,12 +609,14 @@ def VkKeyScanEx(ch, hkl):

def ScreenToClient(hwnd, x, y):
point = POINT(x, y)
user32.ScreenToClient(hwnd, byref(point))
if not user32.ScreenToClient(hwnd, byref(point)):
raise WinError()
return point.x, point.y

def ClientToScreen(hwnd, x, y):
point = POINT(x, y)
user32.ClientToScreen(hwnd, byref(point))
if not user32.ClientToScreen(hwnd, byref(point)):
raise WinError()
return point.x, point.y

def NotifyWinEvent(event, hwnd, idObject, idChild):
@@ -54,14 +54,9 @@ def callback(window, data):
_logicalToPhysicalPoint = ctypes.windll.user32.LogicalToPhysicalPointForPerMonitorDPI
_physicalToLogicalPoint = ctypes.windll.user32.PhysicalToLogicalPointForPerMonitorDPI
except AttributeError:
try:
# Windows Vista..Windows 8
_logicalToPhysicalPoint = ctypes.windll.user32.LogicalToPhysicalPoint
_physicalToLogicalPoint = ctypes.windll.user32.PhysicalToLogicalPoint
except AttributeError:
# Windows <= XP
_logicalToPhysicalPoint = None
_physicalToLogicalPoint = None
# Windows Vista..Windows 8
_logicalToPhysicalPoint = ctypes.windll.user32.LogicalToPhysicalPoint
_physicalToLogicalPoint = ctypes.windll.user32.PhysicalToLogicalPoint

def logicalToPhysicalPoint(window, x, y):
"""Converts the logical coordinates of a point in a window to physical coordinates.
@@ -74,10 +69,11 @@ def logicalToPhysicalPoint(window, x, y):
@return: The physical x and y coordinates.
@rtype: tuple of (int, int)
"""
if not _logicalToPhysicalPoint:
return x, y
point = ctypes.wintypes.POINT(x, y)
_logicalToPhysicalPoint(window, ctypes.byref(point))
if not _logicalToPhysicalPoint(window, ctypes.byref(point)):
raise RuntimeError(
f"Couldn't convert point(x={x}, y={y}) from logical to physical coordinates for window {window}"
)
return point.x, point.y

def physicalToLogicalPoint(window, x, y):
@@ -91,10 +87,11 @@ def physicalToLogicalPoint(window, x, y):
@return: The logical x and y coordinates.
@rtype: tuple of (int, int)
"""
if not _physicalToLogicalPoint:
return x, y
point = ctypes.wintypes.POINT(x, y)
_physicalToLogicalPoint(window, ctypes.byref(point))
if not _physicalToLogicalPoint(window, ctypes.byref(point)):
raise RuntimeError(
f"Couldn't convert point(x={x}, y={y}) from physical to logical coordinates for window {window}"
)
return point.x, point.y

DEFAULT_DPI_LEVEL = 96.0

0 comments on commit 669ae5a

Please sign in to comment.
You can’t perform that action at this time.