Preliminary support for UI Automation in Windows Console

source/NVDAObjects/UIA/__init__.py

@@ -858,6 +858,14 @@ def findOverlayClasses(self,clsList):
 		except ValueError:
 			pass
 
+		# Support Windows Console's UIA interface
+		if (
+			self.windowClassName == "ConsoleWindowClass"
+			and self.UIAElement.cachedAutomationId == "Text Area"
+			and config.conf['UIA']['consoleUIA']
+		):
+			from .winConsoleUIA import winConsoleUIA
+			clsList.append(winConsoleUIA)
 		# Add editableText support if UIA supports a text pattern
 		if self.TextInfo==UIATextInfo:
 			clsList.append(EditableTextWithoutAutoSelectDetection)

source/NVDAObjects/UIA/winConsoleUIA.py

@@ -0,0 +1,62 @@
+# NVDAObjects/UIA/winConsoleUIA.py
+# A part of NonVisual Desktop Access (NVDA)
+# This file is covered by the GNU General Public License.
+# See the file COPYING for more details.
+# Copyright (C) 2019 Bill Dengler
+
+import time
+import textInfos
+import UIAHandler
+
+from scriptHandler import script
+from winVersion import isAtLeastWin10
+from . import UIATextInfo
+from ..behaviors import Terminal
+
+
+class consoleUIATextInfo(UIATextInfo):
+	_expandCollapseBeforeReview = False
+
+	def __init__(self, obj, position, _rangeObj=None):
+		super(consoleUIATextInfo, self).__init__(obj, position, _rangeObj)
+		if position == textInfos.POSITION_CARET:
+			if isAtLeastWin10(1903):
+				# The UIA implementation in 1903 causes the caret to be
+				# off-by-one, so move it one position to the right
+				# to compensate.
+				self._rangeObj.MoveEndpointByUnit(
+					UIAHandler.TextPatternRangeEndpoint_Start,
+					UIAHandler.NVDAUnitsToUIAUnits[textInfos.UNIT_CHARACTER],
+					1
+				)
+
+
+class winConsoleUIA(Terminal):
+	_TextInfo = consoleUIATextInfo
+	_isTyping = False
+	_lastCharTime = 0
+	_TYPING_TIMEOUT = 1
+
+	def _reportNewText(self, line):
+		# Additional typed character filtering beyond that in LiveText
+		if self._isTyping and time.time() - self._lastCharTime <= self._TYPING_TIMEOUT:
+			return
+		super(winConsoleUIA, self)._reportNewText(line)
+
+	def event_typedCharacter(self, ch):
+		if not ch.isspace():
+			self._isTyping = True
+		self._lastCharTime = time.time()
+		super(winConsoleUIA, self).event_typedCharacter(ch)
+
+	@script(gestures=["kb:enter", "kb:numpadEnter", "kb:tab"])
+	def script_clear_isTyping(self, gesture):
+		gesture.send()
+		self._isTyping = False
+
+	def _getTextLines(self):
+		# Filter out extraneous empty lines from UIA
+		# Todo: do this (also) somewhere else so they aren't in document review either
+		ptr = self.UIATextPattern.GetVisibleRanges()
+		res = [ptr.GetElement(i).GetText(-1) for i in range(ptr.length)]
+		return res

source/NVDAObjects/window/__init__.py

@@ -12,6 +12,7 @@
 from logHandler import log
 import controlTypes
 import api
+import config
 import displayModel
 import eventHandler
 from NVDAObjects import NVDAObject
@@ -120,7 +121,7 @@ def findOverlayClasses(self,clsList):
 			from .scintilla import Scintilla as newCls
 		elif windowClassName in ("AkelEditW", "AkelEditA"):
 			from .akelEdit import AkelEdit as newCls
-		elif windowClassName=="ConsoleWindowClass":
+		elif windowClassName=="ConsoleWindowClass" and not config.conf['UIA']['consoleUIA']:
 			from .winConsole import WinConsole as newCls
 		elif windowClassName=="EXCEL7":
 			from .excel import Excel7Window as newCls

source/_UIAHandler.py

@@ -21,6 +21,7 @@
 import NVDAHelper
 import winKernel
 import winUser
+import winVersion
 import eventHandler
 from logHandler import log
 import UIAUtils
@@ -44,28 +45,6 @@
 	'RAIL_WINDOW',
 ]
 
-badUIAWindowClassNames=[
-	"SysTreeView32",
-	"WuDuiListView",
-	"ComboBox",
-	"msctls_progress32",
-	"Edit",
-	"CommonPlacesWrapperWndClass",
-	"SysMonthCal32",
-	"SUPERGRID", #Outlook 2010 message list
-	"RichEdit",
-	"RichEdit20",
-	"RICHEDIT50W",
-	"SysListView32",
-	"EXCEL7",
-	"Button",
-	# #7497: Windows 10 Fall Creators Update has an incomplete UIA implementation for console windows, therefore for now we should ignore it.
-	# It does not implement caret/selection, and probably has no new text events.
-	"ConsoleWindowClass",
-	# #8944: The Foxit UIA implementation is incomplete and should not be used for now.
-	"FoxitDocWnd",
-]
-
 # #8405: used to detect UIA dialogs prior to Windows 10 RS5.
 UIADialogClassNames=[
 	"#32770",
@@ -140,7 +119,6 @@
 
 UIAEventIdsToNVDAEventNames={
 	UIA_LiveRegionChangedEventId:"liveRegionChange",
-	#UIA_Text_TextChangedEventId:"textChanged",
 	UIA_SelectionItem_ElementSelectedEventId:"UIA_elementSelected",
 	UIA_MenuOpenedEventId:"gainFocus",
 	UIA_SelectionItem_ElementAddedToSelectionEventId:"stateChange",
@@ -154,6 +132,9 @@
 	UIA_SystemAlertEventId:"UIA_systemAlert",
 }
 
+if winVersion.isAtLeastWin10():
+	UIAEventIdsToNVDAEventNames[UIA_Text_TextChangedEventId] = "textChange"
+
 class UIAHandler(COMObject):
 	_com_interfaces_=[IUIAutomationEventHandler,IUIAutomationFocusChangedEventHandler,IUIAutomationPropertyChangedEventHandler,IUIAutomationNotificationEventHandler]
 
@@ -342,6 +323,32 @@ def IUIAutomationNotificationEventHandler_HandleNotificationEvent(self,sender,No
 			return
 		eventHandler.queueEvent("UIA_notification",obj, notificationKind=NotificationKind, notificationProcessing=NotificationProcessing, displayString=displayString, activityId=activityId)
 
+	def _isBadUIAWindowClassName(self, windowClass):
+		"Given a windowClassName, returns True if this is a known problematic UIA implementation."
+		badUIAWindowClassNames=[
+			"SysTreeView32",
+			"WuDuiListView",
+			"ComboBox",
+			"msctls_progress32",
+			"Edit",
+			"CommonPlacesWrapperWndClass",
+			"SysMonthCal32",
+			"SUPERGRID", #Outlook 2010 message list
+			"RichEdit",
+			"RichEdit20",
+			"RICHEDIT50W",
+			"SysListView32",
+			"EXCEL7",
+			"Button",
+			# #8944: The Foxit UIA implementation is incomplete and should not be used for now.
+			"FoxitDocWnd",
+		]
+		# #7497: Windows 10 Fall Creators Update has an incomplete UIA implementation for console windows, therefore for now we should ignore it.
+		# It does not implement caret/selection, and probably has no new text events.
+		if not config.conf['UIA']['consoleUIA']:
+			badUIAWindowClassNames.append("ConsoleWindowClass")
+		return windowClass in badUIAWindowClassNames
+
 	def _isUIAWindowHelper(self,hwnd):
 		# UIA in NVDA's process freezes in Windows 7 and below
 		processID=winUser.getWindowThreadProcessID(hwnd)[0]
@@ -358,7 +365,7 @@ def _isUIAWindowHelper(self,hwnd):
 		if appModule and appModule.isGoodUIAWindow(hwnd):
 			return True
 		# There are certain window classes that just had bad UIA implementations
-		if windowClass in badUIAWindowClassNames:
+		if self._isBadUIAWindowClassName(windowClass):
 			return False
 		# allow the appModule for the window to also choose if this window is bad
 		if appModule and appModule.isBadUIAWindow(hwnd):

source/config/configSpec.py

@@ -187,6 +187,7 @@
 [UIA]
 	enabled = boolean(default=true)
 	useInMSWordWhenAvailable = boolean(default=false)
+	consoleUIA = boolean(default=false)
 
 [update]
 	autoCheck = boolean(default=true)

source/globalCommands.py

@@ -983,8 +983,9 @@ def script_review_top(self,gesture):
 
 	def script_review_previousLine(self,gesture):
 		info=api.getReviewPosition().copy()
-		info.expand(textInfos.UNIT_LINE)
-		info.collapse()
+		if info._expandCollapseBeforeReview:
+			info.expand(textInfos.UNIT_LINE)
+			info.collapse()
 		res=info.move(textInfos.UNIT_LINE,-1)
 		if res==0:
 			# Translators: a message reported when review cursor is at the top line of the current navigator object.
@@ -1014,8 +1015,9 @@ def script_review_currentLine(self,gesture):
 
 	def script_review_nextLine(self,gesture):
 		info=api.getReviewPosition().copy()
-		info.expand(textInfos.UNIT_LINE)
-		info.collapse()
+		if info._expandCollapseBeforeReview:
+			info.expand(textInfos.UNIT_LINE)
+			info.collapse()
 		res=info.move(textInfos.UNIT_LINE,1)
 		if res==0:
 			# Translators: a message reported when review cursor is at the bottom line of the current navigator object.
@@ -1041,8 +1043,9 @@ def script_review_bottom(self,gesture):
 
 	def script_review_previousWord(self,gesture):
 		info=api.getReviewPosition().copy()
-		info.expand(textInfos.UNIT_WORD)
-		info.collapse()
+		if info._expandCollapseBeforeReview:
+			info.expand(textInfos.UNIT_WORD)
+			info.collapse()
 		res=info.move(textInfos.UNIT_WORD,-1)
 		if res==0:
 			# Translators: a message reported when review cursor is at the top line of the current navigator object.
@@ -1071,8 +1074,9 @@ def script_review_currentWord(self,gesture):
 
 	def script_review_nextWord(self,gesture):
 		info=api.getReviewPosition().copy()
-		info.expand(textInfos.UNIT_WORD)
-		info.collapse()
+		if info._expandCollapseBeforeReview:
+			info.expand(textInfos.UNIT_WORD)
+			info.collapse()
 		res=info.move(textInfos.UNIT_WORD,1)
 		if res==0:
 			# Translators: a message reported when review cursor is at the bottom line of the current navigator object.
@@ -1087,8 +1091,9 @@ def script_review_nextWord(self,gesture):
 
 	def script_review_startOfLine(self,gesture):
 		info=api.getReviewPosition().copy()
-		info.expand(textInfos.UNIT_LINE)
-		info.collapse()
+		if info._expandCollapseBeforeReview:
+			info.expand(textInfos.UNIT_LINE)
+			info.collapse()
 		api.setReviewPosition(info)
 		info.expand(textInfos.UNIT_CHARACTER)
 		ui.reviewMessage(_("Left"))
@@ -1101,8 +1106,9 @@ def script_review_previousCharacter(self,gesture):
 		lineInfo=api.getReviewPosition().copy()
 		lineInfo.expand(textInfos.UNIT_LINE)
 		charInfo=api.getReviewPosition().copy()
-		charInfo.expand(textInfos.UNIT_CHARACTER)
-		charInfo.collapse()
+		if charInfo._expandCollapseBeforeReview:
+			charInfo.expand(textInfos.UNIT_CHARACTER)
+			charInfo.collapse()
 		res=charInfo.move(textInfos.UNIT_CHARACTER,-1)
 		if res==0 or charInfo.compareEndPoints(lineInfo,"startToStart")<0:
 			# Translators: a message reported when review cursor is at the leftmost character of the current navigator object's text.
@@ -1158,8 +1164,9 @@ def script_review_nextCharacter(self,gesture):
 		lineInfo=api.getReviewPosition().copy()
 		lineInfo.expand(textInfos.UNIT_LINE)
 		charInfo=api.getReviewPosition().copy()
-		charInfo.expand(textInfos.UNIT_CHARACTER)
-		charInfo.collapse()
+		if charInfo._expandCollapseBeforeReview:
+			charInfo.expand(textInfos.UNIT_CHARACTER)
+			charInfo.collapse()
 		res=charInfo.move(textInfos.UNIT_CHARACTER,1)
 		if res==0 or charInfo.compareEndPoints(lineInfo,"endToEnd")>=0:
 			# Translators: a message reported when review cursor is at the rightmost character of the current navigator object's text.

source/gui/settingsDialogs.py

@@ -2050,6 +2050,13 @@ def __init__(self, parent):
 		self.UIAInMSWordCheckBox.SetValue(config.conf["UIA"]["useInMSWordWhenAvailable"])
 		self.UIAInMSWordCheckBox.defaultValue = self._getDefaultValue(["UIA", "useInMSWordWhenAvailable"])
 
+		# Translators: This is the label for a checkbox in the
+		#  Advanced settings panel.
+		label = _("Use UI Automation to access the Windows Console")
+		self.ConsoleUIACheckBox=UIAGroup.addItem(wx.CheckBox(self, label=label))
+		self.ConsoleUIACheckBox.SetValue(config.conf["UIA"]["consoleUIA"])
+		self.ConsoleUIACheckBox.defaultValue = self._getDefaultValue(["UIA", "consoleUIA"])
+
 		# Translators: This is the label for a group of advanced options in the
 		#  Advanced settings panel
 		label = _("Browse mode")
@@ -2152,6 +2159,7 @@ def onSave(self):
 		log.debug("Saving advanced config")
 		config.conf["development"]["enableScratchpadDir"]=self.scratchpadCheckBox.IsChecked()
 		config.conf["UIA"]["useInMSWordWhenAvailable"]=self.UIAInMSWordCheckBox.IsChecked()
+		config.conf["UIA"]["consoleUIA"]=self.ConsoleUIACheckBox.IsChecked()
 		config.conf["virtualBuffers"]["autoFocusFocusableElements"] = self.autoFocusFocusableElementsCheckBox.IsChecked()
 		config.conf["editableText"]["caretMoveTimeoutMs"]=self.caretMoveTimeoutSpinControl.GetValue()
 		for index,key in enumerate(self.logCategories):

source/keyboardHandler.py

@@ -197,18 +197,22 @@ def internal_keyDownEvent(vkCode,scanCode,extended,injected):
 		# #6017: handle typed characters in Win10 RS2 and above where we can't detect typed characters in-process 
 		# This code must be in the 'finally' block as code above returns in several places yet we still want to execute this particular code.
 		focus=api.getFocusObject()
+		from NVDAObjects.UIA.winConsoleUIA import winConsoleUIA
 		if (
 			# This is only possible in Windows 10 RS2 and above
-			winVersion.winVersion.build>=14986
+			winVersion.isAtLeastWin10(1703)
 			# And we only want to do this if the gesture did not result in an executed action 
 			and not gestureExecuted 
 			# and not if this gesture is a modifier key
 			and not isNVDAModifierKey(vkCode,extended) and not vkCode in KeyboardInputGesture.NORMAL_MODIFIER_KEYS
 			and ( # Either of
-				# We couldn't inject in-process, and its not a console window (console windows have their own specific typed character support)
+				# We couldn't inject in-process, and its not a legacy console window.
+				# console windows have their own specific typed character support.
 				(not focus.appModule.helperLocalBindingHandle and focus.windowClassName!='ConsoleWindowClass')
 				# or the focus is within a UWP app, where WM_CHAR never gets sent 
 				or focus.windowClassName.startswith('Windows.UI.Core')
+				#Or this is a UIA console window, where WM_CHAR messages are doubled
+				or isinstance(focus, winConsoleUIA)
 			)
 		):
 			keyStates=(ctypes.c_byte*256)()

source/textInfos/__init__.py

@@ -264,6 +264,10 @@ class TextInfo(baseObject.AutoPropertyObject):
 	@type bookmark: L{Bookmark}
 	"""
 
+	#: whether this textInfo should be expanded then collapsed around its enclosing unit before review.
+	#: This can be problematic for some implementations.
+	_expandCollapseBeforeReview = True
+
 	def __init__(self,obj,position):
 		"""Constructor.
 		Subclasses must extend this, calling the superclass method first.

source/winVersion.py

@@ -26,3 +26,27 @@ def canRunVc2010Builds():
 UWP_OCR_DATA_PATH = os.path.expandvars(r"$windir\OCR")
 def isUwpOcrAvailable():
 	return os.path.isdir(UWP_OCR_DATA_PATH)
+
+def isAtLeastWin10(version=1507):
+	"""
+	Returns True if NVDA is running on at least the supplied version of Windows 10. If no argument is supplied, returns True for all public Windows 10 releases.
+	Note: this function will always return False for source copies of NVDA due to a Python bug.
+	"""
+	from logHandler import log
+	win10VersionsToBuilds={
+		1507: 10240,
+		1511: 10586,
+		1607: 14393,
+		1703: 15063,
+		1709: 16299,
+		1803: 17134,
+		1809: 17763,
+		1903: 18362
+	}
+	if winVersion.major < 10:
+		return False
+	try:
+		return winVersion.build >= win10VersionsToBuilds[version]
+	except KeyError:
+		log.warning("Unknown Windows 10 version {}".format(version))
+		return False

user_docs/en/userGuide.t2t

@@ -1670,6 +1670,9 @@ This includes in Microsoft Word itself, and also the Microsoft Outlook message v
 However, There may be some information which is either not exposed, or exposed incorrectly in some versions of Microsoft Office, which means this UI automation support cannot always be relied upon.
 We still do not recommend that  the majority of users turn this on by default, though we do welcome users of Office 2016/365 to test this feature and provide feedback.
 
+==== Use UI automation to access the Windows Console====[AdvancedSettingsConsoleUIA]
+When this option is enabled, NVDA will use a new, work in progress version of its support for Windows Console which takes advantage of [accessibility improvements made by Microsoft https://devblogs.microsoft.com/commandline/whats-new-in-windows-console-in-windows-10-fall-creators-update/]. This feature is highly experimental and is still incomplete, so its use is not yet recommended. However, once completed, it is anticipated that this new support will become the default, improving NVDA's performance and stability in Windows command consoles.
+
 ==== Automatically set system focus to focusable elements in Browse Mode ====[BrowseModeSettingsAutoFocusFocusableElements]
 Key: NVDA+8