From b4579327bb7a21aa7fe26486451d51ed9f08e857 Mon Sep 17 00:00:00 2001 From: Michael Curran Date: Wed, 2 Aug 2017 11:17:44 +1000 Subject: [PATCH 1/5] Report live region changes outside of web content (E.g. settings search messages, Windows update progress on logon, and Skype for business chat messages etc) (#7287) * A very simple implementation for ARIA live regions outside of web content. UIA's liveRegionChanged event, and MSAA's object_liveRegionChanged event are mapped to a new liveRegionChange event in NVDA. the base NVDAObject handles this event by simply reporting the name of the object in speech and braille. * MSHTML NVDAObject: stub out event_liveRegionChange as MSHTML already has custom live region support via virtualBuffers that is much more accurate. * Automatically report incoming messages in Skype for Business while in a chat window. * Add the EVENT_OBJECT_LIVEREGIONCHANGED MSAA constant to winUser. * For now, do not allow UI Automation live region events with the same text within a short time of each other. Multiple apps in windows 10 (Maps, Store etc) have buggy events. No runtimeID check can be made here as in many cases multiple elements in a list (E.g. store updates for each app) are firing all at the same time. * Added a _shouldAllowUIALiveRegionChangeEvent property to UIA NVDAObject. This is private as it may be changed or removed in the future when live region events are rethought in Windows 10. this property compairs the text and time against the last liveRegion change. --- source/IAccessibleHandler.py | 1 + source/NVDAObjects/IAccessible/MSHTML.py | 4 ++ source/NVDAObjects/UIA/__init__.py | 14 ++++++ source/NVDAObjects/__init__.py | 9 ++++ source/_UIAHandler.py | 7 ++- source/appModules/lync.py | 56 ++++++++++++++++++++++++ source/winUser.py | 2 +- 7 files changed, 91 insertions(+), 2 deletions(-) create mode 100644 source/appModules/lync.py diff --git a/source/IAccessibleHandler.py b/source/IAccessibleHandler.py index 4f678b8bb35..1156cdfe115 100644 --- a/source/IAccessibleHandler.py +++ b/source/IAccessibleHandler.py @@ -488,6 +488,7 @@ def accNavigate(pacc,childID,direction): winUser.EVENT_OBJECT_SELECTIONWITHIN:"selectionWithIn", winUser.EVENT_OBJECT_STATECHANGE:"stateChange", winUser.EVENT_OBJECT_VALUECHANGE:"valueChange", +winUser.EVENT_OBJECT_LIVEREGIONCHANGED:"liveRegionChange", IA2_EVENT_TEXT_CARET_MOVED:"caret", IA2_EVENT_DOCUMENT_LOAD_COMPLETE:"documentLoadComplete", IA2_EVENT_OBJECT_ATTRIBUTE_CHANGED:"IA2AttributeChange", diff --git a/source/NVDAObjects/IAccessible/MSHTML.py b/source/NVDAObjects/IAccessible/MSHTML.py index 89a6d06457d..62e7355fe8a 100644 --- a/source/NVDAObjects/IAccessible/MSHTML.py +++ b/source/NVDAObjects/IAccessible/MSHTML.py @@ -988,6 +988,10 @@ def _get_language(self): except LookupError: return None + def event_liveRegionChange(self): + # MSHTML live regions are currently handled with custom code in-process + pass + class V6ComboBox(IAccessible): """The object which receives value change events for combo boxes in MSHTML/IE 6. """ diff --git a/source/NVDAObjects/UIA/__init__.py b/source/NVDAObjects/UIA/__init__.py index 521d98a6c6a..f4695571e68 100644 --- a/source/NVDAObjects/UIA/__init__.py +++ b/source/NVDAObjects/UIA/__init__.py @@ -10,6 +10,7 @@ from ctypes.wintypes import POINT, RECT from comtypes import COMError from comtypes.automation import VARIANT +import time import weakref import sys import numbers @@ -844,6 +845,19 @@ def _get_shouldAllowUIAFocusEvent(self): except COMError: return True + _lastLiveRegionChangeInfo=(None,None) #: Keeps track of the last live region change (text, time) + def _get__shouldAllowUIALiveRegionChangeEvent(self): + """ + This property decides whether a live region change event should be allowed. It compaires live region event with the last one received, only allowing the event if the text (name) is different, or if the time since the last one is at least 0.5 seconds. + """ + oldText,oldTime=self._lastLiveRegionChangeInfo + newText=self.name + newTime=time.time() + self.__class__._lastLiveRegionChangeInfo=(newText,newTime) + if newText==oldText and oldTime is not None and (newTime-oldTime)<0.5: + return False + return True + def _getUIAPattern(self,ID,interface,cache=False): punk=self.UIAElement.GetCachedPattern(ID) if cache else self.UIAElement.GetCurrentPattern(ID) if punk: diff --git a/source/NVDAObjects/__init__.py b/source/NVDAObjects/__init__.py index e89105f9956..dbae1954977 100644 --- a/source/NVDAObjects/__init__.py +++ b/source/NVDAObjects/__init__.py @@ -16,6 +16,7 @@ from displayModel import DisplayModelTextInfo import baseObject import speech +import ui import api import textInfos.offsets import config @@ -864,6 +865,14 @@ def _reportErrorInPreviousWord(self): import nvwave nvwave.playWaveFile(r"waves\textError.wav") + def event_liveRegionChange(self): + """ + A base implementation for live region change events. + """ + name=self.name + if name: + ui.message(name) + def event_typedCharacter(self,ch): if config.conf["documentFormatting"]["reportSpellingErrors"] and config.conf["keyboard"]["alertForSpellingErrors"] and ( # Not alpha, apostrophe or control. diff --git a/source/_UIAHandler.py b/source/_UIAHandler.py index 6db1e391cd7..95cfccc9ce9 100644 --- a/source/_UIAHandler.py +++ b/source/_UIAHandler.py @@ -126,6 +126,7 @@ } UIAEventIdsToNVDAEventNames={ + UIA_LiveRegionChangedEventId:"liveRegionChange", #UIA_Text_TextChangedEventId:"textChanged", UIA_SelectionItem_ElementSelectedEventId:"UIA_elementSelected", UIA_MenuOpenedEventId:"gainFocus", @@ -223,7 +224,11 @@ def IUIAutomationEventHandler_HandleAutomationEvent(self,sender,eventID): return import NVDAObjects.UIA obj=NVDAObjects.UIA.UIA(UIAElement=sender) - if not obj or (NVDAEventName=="gainFocus" and not obj.shouldAllowUIAFocusEvent): + if ( + not obj + or (NVDAEventName=="gainFocus" and not obj.shouldAllowUIAFocusEvent) + or (NVDAEventName=="liveRegionChange" and not obj._shouldAllowUIALiveRegionChangeEvent) + ): return focus=api.getFocusObject() if obj==focus: diff --git a/source/appModules/lync.py b/source/appModules/lync.py new file mode 100644 index 00000000000..9282a45eb21 --- /dev/null +++ b/source/appModules/lync.py @@ -0,0 +1,56 @@ +#A part of NonVisual Desktop Access (NVDA) +#This file is covered by the GNU General Public License. +#See the file COPYING for more details. +#Copyright (C) 2017 NV Access Limited + +"""appModule for Microsoft Skype for business. """ + +import ui +from NVDAObjects.UIA import UIA +import appModuleHandler + +class NetUIRicherLabel(UIA): + """A label sometimes found within list items that can fire live region changes, such as for chat messages.""" + + def event_liveRegionChange(self): + # The base liveRegionChange event is not enough as Skype for Business concatinates recent chat messages from the same person within the same minute + # Therefore, specifically strip out the chat content and only report the most recent part added. + # The object's name contains the full message (I.e. person: content, timestamp) loosely separated by commas. + # Example string: "Michael Curran : , , Hello\r\n\r\nThis is a test , 10:45 am." + # Where person is "Michael Curran", content is "Hello\nThis is a test" and timestamp is "10:45 am" + # The object's value just contains the content. + # Example: "Hello\rThis is a test" + # We are only interested in person and content + # Therefore use value (content) to locate and split off the person from the name (fullText) + # Normalize the usage of end-of-line characters (name and value seem to expose them differently, which would break comparison) + content=self.value.replace('\r','\n').strip() + fullText=self.name.replace('\r\n\r\n','\n') + contentLines=content.split('\n') + contentStartIndex=fullText.find(content) + pretext=fullText[:contentStartIndex] + # There are some annoying comma characters after the person's name + pretext=pretext.replace(' ,','') + # If the objects are the same, the person is the same, and the new content is the old content but with more appended, report the appended content + # Otherwise, report the person and the initial content + runtimeID=self.UIAElement.getRuntimeId() + lastRuntimeID,lastPretext,lastContentLines=self.appModule._lastLiveChatMessageData + contentLinesLen=len(contentLines) + lastContentLinesLen=len(lastContentLines) + if runtimeID==lastRuntimeID and pretext==lastPretext and contentLinesLen>lastContentLinesLen and contentLines[:lastContentLinesLen]==lastContentLines: + message="\n".join(contentLines[lastContentLinesLen:]) + else: + message=pretext+content + ui.message(message) + # Cache the message data for later possible comparisons + self.appModule._lastLiveChatMessageData=runtimeID,pretext,contentLines + +class AppModule(appModuleHandler.AppModule): + + # data to store the last chat message (runtime ID,person,content lines) + _lastLiveChatMessageData=[],"",[] + + def chooseNVDAObjectOverlayClasses(self,obj,clsList): + if isinstance(obj,UIA) and obj.UIAElement.cachedClassName=='NetUIRicherLabel': + clsList.insert(0,NetUIRicherLabel) + return clsList + diff --git a/source/winUser.py b/source/winUser.py index 7dda83331b9..e7b2b4712f4 100644 --- a/source/winUser.py +++ b/source/winUser.py @@ -265,7 +265,7 @@ class GUITHREADINFO(Structure): EVENT_OBJECT_HELPCHANGE=0x8010 EVENT_OBJECT_DEFACTIONCHANGE=0x8011 EVENT_OBJECT_ACCELERATORCHANGE=0x8012 - +EVENT_OBJECT_LIVEREGIONCHANGED=0x8019 EVENT_SYSTEM_DESKTOPSWITCH=0x20 EVENT_OBJECT_INVOKED=0x8013 EVENT_OBJECT_TEXTSELECTIONCHANGED=0x8014 From 82c332e9553ee444fd8f405bc27261fd8e0ac6b3 Mon Sep 17 00:00:00 2001 From: Michael Curran Date: Wed, 2 Aug 2017 11:21:54 +1000 Subject: [PATCH 2/5] Update What's new. --- user_docs/en/changes.t2t | 1 + 1 file changed, 1 insertion(+) diff --git a/user_docs/en/changes.t2t b/user_docs/en/changes.t2t index fbcb3934ed2..b1be9c90ec8 100644 --- a/user_docs/en/changes.t2t +++ b/user_docs/en/changes.t2t @@ -18,6 +18,7 @@ - The currently selected suggestion is now reported in Windows 10 Mail to/cc fields and the Windows 10 Settings search field. (#6241) - A sound is now playd to indicate the appearance of suggestions in certain search fields in Windows 10 (E.g. start screen, settings search, Windows 10 mail to/cc fields). (#6241) - Automatically report notifications in Skype for Business Desktop, such as when someone starts a conversation with you. (#7281) +- Automatically report incoming chat messages while in a Skype for Business conversation. (#7286) - Automatically report notifications in Microsoft Edge, such as when a download starts. (#7281) - You can now type in both contracted and uncontracted braille on a braille display with a braille keyboard. See the Braille Input section of the User Guide for details. (#2439) - You can now enter Unicode braille characters from the braille keyboard on a braille display by selecting Unicode braille as the input table in Braille Settings. (#6449) From 393b55b30582615d68dc109d463de9c7205826f9 Mon Sep 17 00:00:00 2001 From: Michael Curran Date: Wed, 2 Aug 2017 11:46:40 +1000 Subject: [PATCH 3/5] Support ARIA table attributes, skip layout tables, and don't get stuck on hidden cells (#7410) * Add support for ARIA rowindex, colindex, rowcount and colcount in Gecko / Chrome for both browse mode and focus mode. Specifically: * IAccessible NVDAObject properties: rowNumber, columnNumber, rowCount and columnCount now expose the ARIA values if available, otherwise falling back to the original physical table information. * Gecko controlField attributes: table-rownumber, table-columnnumber, table-rowcount and table-columncount now expose the ARIA values if available, otherwise falling back to the original physical table information * New Gecko controlField attributes: table-physicalrownumber, table-physicalcolumnnumber, table-physicalrowcount and table-physicalcolumncount values always expose the physical table information no matter what the presentational values may be. * Added new class variables to BrowseModeDocumentTreeInterceptor: navigationalTableRowNumberAttributeName and navigationalTableColumnNumberAttributeName which are used by BrowseModeDocumentTreeInterceptor._getTableCellCoords in place of the literal strings "table-rownumber" and "table-columnnumber" respectively, to allow fetching of the correct attributes containing physical table information. * Gecko_ia2 VirtualBuffer: override navigationalTableRowNumberAttributeName and navigationalTableColumnNumberAttributeName To provide the use of table-physicalrownumber and table-physicalcolumnnumber. * No longer include layout tables in table navigation commands * Gecko browseMode: skip over hidden table cells in table navigation commands. MSHTML already did this. --- .../vbufBackends/gecko_ia2/gecko_ia2.cpp | 40 ++++++++++++++-- nvdaHelper/vbufBackends/gecko_ia2/gecko_ia2.h | 2 +- source/NVDAObjects/IAccessible/__init__.py | 36 ++++++++++++-- source/browseMode.py | 48 +++++++++++++++---- source/virtualBuffers/gecko_ia2.py | 13 +++++ 5 files changed, 121 insertions(+), 18 deletions(-) diff --git a/nvdaHelper/vbufBackends/gecko_ia2/gecko_ia2.cpp b/nvdaHelper/vbufBackends/gecko_ia2/gecko_ia2.cpp index 2a07a4e4783..b370609a1cd 100755 --- a/nvdaHelper/vbufBackends/gecko_ia2/gecko_ia2.cpp +++ b/nvdaHelper/vbufBackends/gecko_ia2/gecko_ia2.cpp @@ -78,13 +78,18 @@ IAccessible2* IAccessible2FromIdentifier(int docHandle, int ID) { template inline void fillTableCounts(VBufStorage_controlFieldNode_t* node, IAccessible2* pacc, TableType* paccTable) { wostringstream s; long count = 0; + // Fetch row and column counts and add them as two sets of attributes on this vbuf node. + // The first set: table-physicalrowcount and table-physicalcolumncount represent the physical topology of the table and can be used programmatically to understand table limits. + // The second set: table-rowcount and table-columncount are duplicates of the physical ones, however may be overridden later on in fillVBuf with ARIA attributes. They are what is reported to the user. if (paccTable->get_nRows(&count) == S_OK) { s << count; + node->addAttribute(L"table-physicalrowcount", s.str()); node->addAttribute(L"table-rowcount", s.str()); s.str(L""); } if (paccTable->get_nColumns(&count) == S_OK) { s << count; + node->addAttribute(L"table-physicalcolumncount", s.str()); node->addAttribute(L"table-columncount", s.str()); } } @@ -128,11 +133,17 @@ inline void fillTableCellInfo_IATable(VBufStorage_controlFieldNode_t* node, IAcc long cellIndex = _wtoi(cellIndexStr.c_str()); long row, column, rowExtents, columnExtents; boolean isSelected; + // Fetch row and column extents and add them as attributes on this node. + // for rowNumber and columnNumber, store these as two sets of attributes. + // The first set: table-physicalrownumber and table-physicalcolumnnumber represent the physical topology of the table and can be used programmatically to fetch other table cells with IAccessibleTable etc. + // The second set: table-rownumber and table-columnnumber are duplicates of the physical ones, however may be overridden later on in fillVBuf with ARIA attributes. They are what is reported to the user. if (paccTable->get_rowColumnExtentsAtIndex(cellIndex, &row, &column, &rowExtents, &columnExtents, &isSelected) == S_OK) { s << row + 1; + node->addAttribute(L"table-physicalrownumber", s.str()); node->addAttribute(L"table-rownumber", s.str()); s.str(L""); s << column + 1; + node->addAttribute(L"table-physicalcolumnnumber", s.str()); node->addAttribute(L"table-columnnumber", s.str()); if (columnExtents > 1) { s.str(L""); @@ -186,11 +197,17 @@ inline void GeckoVBufBackend_t::fillTableCellInfo_IATable2(VBufStorage_controlFi long row, column, rowExtents, columnExtents; boolean isSelected; + // Fetch row and column extents and add them as attributes on this node. + // for rowNumber and columnNumber, store these as two sets of attributes. + // The first set: table-physicalrownumber and table-physicalcolumnnumber represent the physical topology of the table and can be used programmatically to fetch other table cells with IAccessibleTable etc. + // The second set: table-rownumber and table-columnnumber are duplicates of the physical ones, however may be overridden later on in fillVBuf with ARIA attributes. They are what is reported to the user. if (paccTableCell->get_rowColumnExtents(&row, &column, &rowExtents, &columnExtents, &isSelected) == S_OK) { s << row + 1; + node->addAttribute(L"table-physicalrownumber", s.str()); node->addAttribute(L"table-rownumber", s.str()); s.str(L""); s << column + 1; + node->addAttribute(L"table-physicalcolumnnumber", s.str()); node->addAttribute(L"table-columnnumber", s.str()); if (columnExtents > 1) { s.str(L""); @@ -305,7 +322,7 @@ const wregex REGEX_PRESENTATION_ROLE(L"IAccessible2\\\\:\\\\:attribute_xml-roles VBufStorage_fieldNode_t* GeckoVBufBackend_t::fillVBuf(IAccessible2* pacc, VBufStorage_buffer_t* buffer, VBufStorage_controlFieldNode_t* parentNode, VBufStorage_fieldNode_t* previousNode, - IAccessibleTable* paccTable, IAccessibleTable2* paccTable2, long tableID, + IAccessibleTable* paccTable, IAccessibleTable2* paccTable2, long tableID, const wchar_t* parentPresentationalRowNumber, bool ignoreInteractiveUnlabelledGraphics ) { nhAssert(buffer); //buffer can't be NULL @@ -655,6 +672,23 @@ VBufStorage_fieldNode_t* GeckoVBufBackend_t::fillVBuf(IAccessible2* pacc, } } + // Add some presentational table attributes + // Note these are only for reporting, the physical table attributes (table-physicalrownumber etc) for aiding in navigation etc are added later on. + // propagate table-rownumber down to the cell as Gecko only includes it on the row itself + if(parentPresentationalRowNumber) + parentNode->addAttribute(L"table-rownumber",parentPresentationalRowNumber); + const wchar_t* presentationalRowNumber=NULL; + if((IA2AttribsMapIt = IA2AttribsMap.find(L"rowindex")) != IA2AttribsMap.end()) { + parentNode->addAttribute(L"table-rownumber",IA2AttribsMapIt->second); + presentationalRowNumber=IA2AttribsMapIt->second.c_str(); + } + if((IA2AttribsMapIt = IA2AttribsMap.find(L"colindex")) != IA2AttribsMap.end()) + parentNode->addAttribute(L"table-columnnumber",IA2AttribsMapIt->second); + if((IA2AttribsMapIt = IA2AttribsMap.find(L"rowcount")) != IA2AttribsMap.end()) + parentNode->addAttribute(L"table-rowcount",IA2AttribsMapIt->second); + if((IA2AttribsMapIt = IA2AttribsMap.find(L"colcount")) != IA2AttribsMap.end()) + parentNode->addAttribute(L"table-columncount",IA2AttribsMapIt->second); + BSTR value=NULL; if(pacc->get_accValue(varChild,&value)==S_OK) { if(value&&SysStringLen(value)==0) { @@ -736,7 +770,7 @@ VBufStorage_fieldNode_t* GeckoVBufBackend_t::fillVBuf(IAccessible2* pacc, continue; } paccHyperlink->Release(); - if (tempNode = this->fillVBuf(childPacc, buffer, parentNode, previousNode, paccTable, paccTable2, tableID, ignoreInteractiveUnlabelledGraphics)) { + if (tempNode = this->fillVBuf(childPacc, buffer, parentNode, previousNode, paccTable, paccTable2, tableID, presentationalRowNumber, ignoreInteractiveUnlabelledGraphics)) { previousNode=tempNode; } else { LOG_DEBUG(L"Error in fillVBuf"); @@ -768,7 +802,7 @@ VBufStorage_fieldNode_t* GeckoVBufBackend_t::fillVBuf(IAccessible2* pacc, VariantClear(&(varChildren[i])); continue; } - if (tempNode = this->fillVBuf(childPacc, buffer, parentNode, previousNode, paccTable, paccTable2, tableID, ignoreInteractiveUnlabelledGraphics)) + if (tempNode = this->fillVBuf(childPacc, buffer, parentNode, previousNode, paccTable, paccTable2, tableID, presentationalRowNumber, ignoreInteractiveUnlabelledGraphics)) previousNode=tempNode; else LOG_DEBUG(L"Error in calling fillVBuf"); diff --git a/nvdaHelper/vbufBackends/gecko_ia2/gecko_ia2.h b/nvdaHelper/vbufBackends/gecko_ia2/gecko_ia2.h index b39de514f1e..f488737ad95 100755 --- a/nvdaHelper/vbufBackends/gecko_ia2/gecko_ia2.h +++ b/nvdaHelper/vbufBackends/gecko_ia2/gecko_ia2.h @@ -22,7 +22,7 @@ class GeckoVBufBackend_t: public VBufBackend_t { VBufStorage_fieldNode_t* fillVBuf(IAccessible2* pacc, VBufStorage_buffer_t* buffer, VBufStorage_controlFieldNode_t* parentNode, VBufStorage_fieldNode_t* previousNode, - IAccessibleTable* paccTable=NULL, IAccessibleTable2* paccTable2=NULL, long tableID=0, + IAccessibleTable* paccTable=NULL, IAccessibleTable2* paccTable2=NULL, long tableID=0, const wchar_t* parentPresentationalRowNumber=NULL, bool ignoreInteractiveUnlabelledGraphics=false ); diff --git a/source/NVDAObjects/IAccessible/__init__.py b/source/NVDAObjects/IAccessible/__init__.py index 5a1bdaac1cc..8dafaebd122 100644 --- a/source/NVDAObjects/IAccessible/__init__.py +++ b/source/NVDAObjects/IAccessible/__init__.py @@ -1027,6 +1027,8 @@ def getChild(self, index): return self.correctAPIForRelation(IAccessible(IAccessibleObject=child[0], IAccessibleChildID=child[1])) def _get_IA2Attributes(self): + if not isinstance(self.IAccessibleObject,IAccessibleHandler.IAccessible2): + return {} try: attribs = self.IAccessibleObject.attributes except COMError as e: @@ -1040,7 +1042,7 @@ def event_IA2AttributeChange(self): # We currently only care about changes to the accessible drag and drop attributes, which we map to states, so treat this as a stateChange. self.event_stateChange() - def _get_rowNumber(self): + def _get_IA2PhysicalRowNumber(self): table=self.table if table: if self.IAccessibleTableUsesTableCellIndexAttrib: @@ -1057,7 +1059,15 @@ def _get_rowNumber(self): log.debugWarning("IAccessibleTable::rowIndex failed", exc_info=True) raise NotImplementedError - def _get_columnNumber(self): + def _get_rowNumber(self): + index=self.IA2Attributes.get('rowindex') + if index is None and isinstance(self.parent,IAccessible): + index=self.parent.IA2Attributes.get('rowindex') + if index is None: + index=self.IA2PhysicalRowNumber + return index + + def _get_IA2PhysicalColumnNumber(self): table=self.table if table: if self.IAccessibleTableUsesTableCellIndexAttrib: @@ -1074,7 +1084,13 @@ def _get_columnNumber(self): log.debugWarning("IAccessibleTable::columnIndex failed", exc_info=True) raise NotImplementedError - def _get_rowCount(self): + def _get_columnNumber(self): + index=self.IA2Attributes.get('colindex') + if index is None: + index=self.IA2PhysicalColumnNumber + return index + + def _get_IA2PhysicalRowCount(self): if hasattr(self,'IAccessibleTableObject'): try: return self.IAccessibleTableObject.nRows @@ -1082,7 +1098,13 @@ def _get_rowCount(self): log.debugWarning("IAccessibleTable::nRows failed", exc_info=True) raise NotImplementedError - def _get_columnCount(self): + def _get_rowCount(self): + count=self.IA2Attributes.get('rowcount') + if count is None: + count=self.IA2PhysicalRowCount + return count + + def _get_IA2PhysicalColumnCount(self): if hasattr(self,'IAccessibleTableObject'): try: return self.IAccessibleTableObject.nColumns @@ -1090,6 +1112,12 @@ def _get_columnCount(self): log.debugWarning("IAccessibleTable::nColumns failed", exc_info=True) raise NotImplementedError + def _get_columnCount(self): + count=self.IA2Attributes.get('colcount') + if count is None: + count=self.IA2PhysicalColumnCount + return count + def _get__IATableCell(self): # Permanently cache the result. try: diff --git a/source/browseMode.py b/source/browseMode.py index 0a46851a8dc..7df9e78cf2a 100644 --- a/source/browseMode.py +++ b/source/browseMode.py @@ -1539,6 +1539,11 @@ def script_movePastEndOfContainer(self,gesture): # Translators: Description for the Move past end of container command in browse mode. script_movePastEndOfContainer.__doc__=_("Moves past the end of the container element, such as a list or table") + #: The controlField attribute name that should be used as the row number when navigating in a table. By default this is the same as the presentational attribute name + navigationalTableRowNumberAttributeName="table-rownumber" + #: The controlField attribute name that should be used as the column number when navigating in a table. By default this is the same as the presentational attribute name + navigationalTableColumnNumberAttributeName="table-columnnumber" + def _getTableCellCoords(self, info): """ Fetches information about the deepest table cell at the given position. @@ -1551,17 +1556,28 @@ def _getTableCellCoords(self, info): if info.isCollapsed: info = info.copy() info.expand(textInfos.UNIT_CHARACTER) - for field in reversed(info.getTextWithFields()): + fields=list(info.getTextWithFields()) + # First record the ID of all layout tables so that we can skip them when searching for the deepest table + layoutIDs=set() + for field in fields: + if isinstance(field, textInfos.FieldCommand) and field.command == "controlStart" and field.field.get('table-layout'): + tableID=field.field.get('table-id') + if tableID is not None: + layoutIDs.add(tableID) + for field in reversed(fields): if not (isinstance(field, textInfos.FieldCommand) and field.command == "controlStart"): # Not a control field. continue attrs = field.field - if "table-id" in attrs and "table-rownumber" in attrs: + tableID=attrs.get('table-id') + if tableID is None or tableID in layoutIDs: + continue + if self.navigationalTableColumnNumberAttributeName in attrs and not attrs.get('table-layout'): break else: raise LookupError("Not in a table cell") return (attrs["table-id"], - attrs["table-rownumber"], attrs["table-columnnumber"], + attrs[self.navigationalTableRowNumberAttributeName], attrs[self.navigationalTableColumnNumberAttributeName], attrs.get("table-rowsspanned", 1), attrs.get("table-columnsspanned", 1)) def _getTableCellAt(self,tableID,startPos,row,column): @@ -1576,13 +1592,16 @@ def _getTableCellAt(self,tableID,startPos,row,column): @type column: int @returns: the table cell's position in the document @rtype: L{textInfos.TextInfo} + @raises: LookupError if the cell does not exist """ raise NotImplementedError + _missingTableCellSearchLimit=3 #: The number of missing cells L{_getNearestTableCell} is allowed to skip over to locate the next available cell def _getNearestTableCell(self, tableID, startPos, origRow, origCol, origRowSpan, origColSpan, movement, axis): """ Locates the nearest table cell relative to another table cell in a given direction, given its coordinates. For example, this is used to move to the cell in the next column, previous row, etc. + This method will skip over missing table cells (where L{_getTableCellAt} raises LookupError), up to the number of times set by _missingTableCellSearchLimit set on this instance. @param tableID: the ID of the table @param startPos: the position in the document to start searching from. @type startPos: L{textInfos.TextInfo} @@ -1612,19 +1631,28 @@ def _getNearestTableCell(self, tableID, startPos, origRow, origCol, origRowSpan, elif axis == "column": destCol += origColSpan if movement == "next" else -1 - if destCol < 1 or destRow<1: - # Optimisation: We're definitely at the edge of the column or row. - raise LookupError - - return self._getTableCellAt(tableID,startPos,destRow,destCol) + # Try and fetch the cell at these coordinates, though if a cell is missing, try several more times moving the coordinates on by one cell each time + limit=self._missingTableCellSearchLimit + while limit>0: + limit-=1 + if destCol < 1 or destRow<1: + # Optimisation: We're definitely at the edge of the column or row. + raise LookupError + try: + return self._getTableCellAt(tableID,startPos,destRow,destCol) + except LookupError: + pass + if axis=="row": + destRow+=1 if movement=="next" else -1 + else: + destCol+=1 if movement=="next" else -1 + raise LookupError def _tableMovementScriptHelper(self, movement="next", axis=None): if isScriptWaiting(): return formatConfig=config.conf["documentFormatting"].copy() formatConfig["reportTables"]=True - # For now, table movement includes layout tables even if reporting of layout tables is disabled. - formatConfig["includeLayoutTables"]=True try: tableID, origRow, origCol, origRowSpan, origColSpan = self._getTableCellCoords(self.selection) except LookupError: diff --git a/source/virtualBuffers/gecko_ia2.py b/source/virtualBuffers/gecko_ia2.py index 0cb5f837d0b..c76e0f2e8b8 100755 --- a/source/virtualBuffers/gecko_ia2.py +++ b/source/virtualBuffers/gecko_ia2.py @@ -23,6 +23,11 @@ class Gecko_ia2_TextInfo(VirtualBufferTextInfo): def _normalizeControlField(self,attrs): + for attr in ("table-physicalrownumber","table-physicalcolumnnumber","table-physicalrowcount","table-physicalcolumncount"): + attrVal=attrs.get(attr) + if attrVal is not None: + attrs[attr]=int(attrVal) + current = attrs.get("IAccessible2::attribute_current") if current is not None: attrs['current']= current @@ -260,12 +265,20 @@ def event_scrollingStart(self, obj, nextHandler): return nextHandler() event_scrollingStart.ignoreIsReady = True + # NVDA exposes IAccessible2 table interface row and column numbers as table-physicalrownumber and table-physicalcolumnnumber respectively. + # These should be used when navigating the physical table (I.e. these values should be provided to the table interfaces). + # The presentational table-columnnumber and table-rownumber attributes are normally duplicates of the physical ones, but are overridden by the values of aria-rowindex and aria-colindex if present. + navigationalTableRowNumberAttributeName="table-physicalrownumber" + navigationalTableColumnNumberAttributeName="table-physicalcolumnnumber" + def _getTableCellAt(self,tableID,startPos,destRow,destCol): docHandle = self.rootDocHandle table = self.getNVDAObjectFromIdentifier(docHandle, tableID) try: cell = table.IAccessibleTableObject.accessibleAt(destRow - 1, destCol - 1).QueryInterface(IAccessible2) cell = NVDAObjects.IAccessible.IAccessible(IAccessibleObject=cell, IAccessibleChildID=0) + if cell.IA2Attributes.get('hidden'): + raise LookupError("Found hidden cell") return self.makeTextInfo(cell) except (COMError, RuntimeError): raise LookupError From 6371163005b840269f5aeeb15e280342557f7f9f Mon Sep 17 00:00:00 2001 From: Michael Curran Date: Wed, 2 Aug 2017 12:04:40 +1000 Subject: [PATCH 4/5] Update what's new --- user_docs/en/changes.t2t | 3 +++ 1 file changed, 3 insertions(+) diff --git a/user_docs/en/changes.t2t b/user_docs/en/changes.t2t index b1be9c90ec8..ec54cb9f08f 100644 --- a/user_docs/en/changes.t2t +++ b/user_docs/en/changes.t2t @@ -33,6 +33,7 @@ - For example, the "Fill display for context changes" and "Only when scrolling back" options can make working with lists and menus more efficient, since the items won't continually change their position on the display. - See the section on the "Focus context presentation" setting in the User Guide for further details and examples. - Many more control types and states have been abbreviated for braille. Please see "Control Type, State and Landmark Abbreviations" under Braille in the User Guide for a complete list. (#7188) +- In Firefox and Chrome, NVDA now supports complex dynamic grids such as spreadsheets where only some of the content might be loaded or displayed (specifically, the aria-rowcount, aria-colcount, aria-rowindex and aria-colindex attributes introduced in ARIA 1.1). (#7410) == Changes == @@ -80,6 +81,8 @@ - In Microsoft Edge, landmarks are correctly localized in languages other than English. (#7328) - Braille now correctly follows the selection when selecting text beyond the width of the display. For example, if you select multiple lines with shift+downArrow, braille now shows the last line you selected. (#5770) - In Firefox, NVDA no longer spuriously reports "section" several times when opening details for a tweet on twitter.com. (#5741) +- Table navigation commands are no longer available for layout tables in Browse Mode unless reporting of layout tables is enabled. (#7382) +- In Firefox and Chrome, Browse Mode table navigation commands now skip over hidden table cells. (#6652, #5655) == Changes for Developers == From 99ef8a82f17e5d6625aef3499ae6891d1ac72938 Mon Sep 17 00:00:00 2001 From: Michael Curran Date: Wed, 2 Aug 2017 12:24:05 +1000 Subject: [PATCH 5/5] Still enable table navigation for layout tables if the user has configured layout tables to be reported. --- source/browseMode.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/source/browseMode.py b/source/browseMode.py index 7df9e78cf2a..1aa58976aaa 100644 --- a/source/browseMode.py +++ b/source/browseMode.py @@ -1557,13 +1557,14 @@ def _getTableCellCoords(self, info): info = info.copy() info.expand(textInfos.UNIT_CHARACTER) fields=list(info.getTextWithFields()) - # First record the ID of all layout tables so that we can skip them when searching for the deepest table + # If layout tables should not be reported, we should First record the ID of all layout tables so that we can skip them when searching for the deepest table layoutIDs=set() - for field in fields: - if isinstance(field, textInfos.FieldCommand) and field.command == "controlStart" and field.field.get('table-layout'): - tableID=field.field.get('table-id') - if tableID is not None: - layoutIDs.add(tableID) + if not config.conf["documentFormatting"]["includeLayoutTables"]: + for field in fields: + if isinstance(field, textInfos.FieldCommand) and field.command == "controlStart" and field.field.get('table-layout'): + tableID=field.field.get('table-id') + if tableID is not None: + layoutIDs.add(tableID) for field in reversed(fields): if not (isinstance(field, textInfos.FieldCommand) and field.command == "controlStart"): # Not a control field.