From 33ce9fb299661844b8a17bde3145026a40344760 Mon Sep 17 00:00:00 2001 From: Michael Curran Date: Mon, 17 Oct 2022 16:39:24 +1000 Subject: [PATCH 01/10] UIAHandler propertyChange event: avoid needlessly creating an NVDAObject when the event is for the focus object. --- source/UIAHandler/__init__.py | 57 ++++++++++++++++++++--------------- 1 file changed, 32 insertions(+), 25 deletions(-) diff --git a/source/UIAHandler/__init__.py b/source/UIAHandler/__init__.py index 607528c8230..30621cd1e65 100644 --- a/source/UIAHandler/__init__.py +++ b/source/UIAHandler/__init__.py @@ -909,6 +909,7 @@ def IUIAutomationPropertyChangedEventHandler_HandlePropertyChangedEvent(self,sen if _isDebug(): log.debugWarning(f"HandlePropertyChangedEvent: Don't know how to handle property {propertyId}") return + obj = None focus = api.getFocusObject() import NVDAObjects.UIA if ( @@ -916,42 +917,48 @@ def IUIAutomationPropertyChangedEventHandler_HandlePropertyChangedEvent(self,sen and self.clientObject.compareElements(focus.UIAElement, sender) ): if _isDebug(): - log.debug("propertyChange event is for focus") - pass + log.debug( + "propertyChange event is for focus. " + f"Redirecting event to focus NVDAObject {focus}" + ) + obj = focus elif not self.isNativeUIAElement(sender): if _isDebug(): log.debug( f"HandlePropertyChangedEvent: Ignoring event {NVDAEventName} for non native element" ) return - window = self.getNearestWindowHandle(sender) - if window and not eventHandler.shouldAcceptEvent(NVDAEventName, windowHandle=window): + window = obj.windowHandle if obj else self.getNearestWindowHandle(sender) + if window: if _isDebug(): log.debug( - f"HandlePropertyChangedEvent: Ignoring event {NVDAEventName} for shouldAcceptEvent=False" - ) - return - try: - obj = NVDAObjects.UIA.UIA(UIAElement=sender) - except Exception: - if _isDebug(): - log.debugWarning( - f"HandlePropertyChangedEvent: Exception while creating object for event {NVDAEventName}", - exc_info=True + f"Checking if should accept NVDA event {NVDAEventName} " + f"with window {self.getWindowHandleDebugString(window)}" ) - return + if not eventHandler.shouldAcceptEvent(NVDAEventName, windowHandle=window): + if _isDebug(): + log.debug( + f"HandlePropertyChangedEvent: Ignoring event {NVDAEventName} for shouldAcceptEvent=False" + ) + return if not obj: + try: + obj = NVDAObjects.UIA.UIA(UIAElement=sender) + except Exception: + if _isDebug(): + log.debugWarning( + f"HandlePropertyChangedEvent: Exception while creating object for event {NVDAEventName}", + exc_info=True + ) + return + if not obj: + if _isDebug(): + log.debug(f"HandlePropertyChangedEvent: Ignoring event {NVDAEventName} because no object") + return if _isDebug(): - log.debug(f"HandlePropertyChangedEvent: Ignoring event {NVDAEventName} because no object") - return - if _isDebug(): - log.debug( - f"handlePropertyChangeEvent: created object {obj} " - ) - if obj==focus: - if _isDebug(): - log.debug("handlePropertyChangeEvent: redirecting to focus") - obj=focus + log.debug( + f"handlePropertyChangeEvent: created object {obj} " + ) if _isDebug(): log.debug( f"handlePropertyChangeEvent: queuing NVDA {NVDAEventName} event " From d209f17d58c4ef2caaf4fa47866e5bcf01877750 Mon Sep 17 00:00:00 2001 From: Michael Curran Date: Mon, 17 Oct 2022 16:53:06 +1000 Subject: [PATCH 02/10] UIAHandler automation event: don't needlessly create an NVDAObject when event is for the focus. --- source/UIAHandler/__init__.py | 57 ++++++++++++++++++++--------------- 1 file changed, 32 insertions(+), 25 deletions(-) diff --git a/source/UIAHandler/__init__.py b/source/UIAHandler/__init__.py index 30621cd1e65..6a6ed75d1eb 100644 --- a/source/UIAHandler/__init__.py +++ b/source/UIAHandler/__init__.py @@ -730,6 +730,7 @@ def IUIAutomationEventHandler_HandleAutomationEvent(self,sender,eventID): if _isDebug(): log.debugWarning(f"HandleAutomationEvent: Don't know how to handle event {eventID}") return + obj = None focus = api.getFocusObject() import NVDAObjects.UIA if ( @@ -737,38 +738,48 @@ def IUIAutomationEventHandler_HandleAutomationEvent(self,sender,eventID): and self.clientObject.compareElements(focus.UIAElement, sender) ): if _isDebug(): - log.debug("handleAutomationEvent: element matches focus") - pass + log.debug( + "handleAutomationEvent: element matches focus. " + f"Redirecting event to focus NVDAObject {focus}" + ) + obj = focus elif not self.isNativeUIAElement(sender): if _isDebug(): log.debug( f"HandleAutomationEvent: Ignoring event {NVDAEventName} for non native element" ) return - window = self.getNearestWindowHandle(sender) - if window and not eventHandler.shouldAcceptEvent(NVDAEventName, windowHandle=window): + window = obj.windowHandle if obj else self.getNearestWindowHandle(sender) + if window: if _isDebug(): log.debug( - f"HandleAutomationEvent: Ignoring event {NVDAEventName} for shouldAcceptEvent=False" - ) - return - try: - obj = NVDAObjects.UIA.UIA(UIAElement=sender) - except Exception: - if _isDebug(): - log.debugWarning( - f"HandleAutomationEvent: Exception while creating object for event {NVDAEventName}", - exc_info=True + f"Checking if should accept NVDA event {NVDAEventName} " + f"with window {self.getWindowHandleDebugString(window)}" ) - return + if not eventHandler.shouldAcceptEvent(NVDAEventName, windowHandle=window): + if _isDebug(): + log.debug( + f"HandleAutomationEvent: Ignoring event {NVDAEventName} for shouldAcceptEvent=False" + ) + return if not obj: - if _isDebug(): - log.debug("handleAutomationEvent: No NVDAObject could be created") + try: + obj = NVDAObjects.UIA.UIA(UIAElement=sender) + except Exception: + if _isDebug(): + log.debugWarning( + f"HandleAutomationEvent: Exception while creating object for event {NVDAEventName}", + exc_info=True + ) return - if _isDebug(): - log.debug( - f"handleAutomationEvent: created object {obj} " - ) + if not obj: + if _isDebug(): + log.debug(f"handleAutomationEvent: No NVDAObject could be created") + return + if _isDebug(): + log.debug( + f"handleAutomationEvent: created object {obj} " + ) if ( (NVDAEventName == "gainFocus" and not obj.shouldAllowUIAFocusEvent) or (NVDAEventName=="liveRegionChange" and not obj._shouldAllowUIALiveRegionChangeEvent) @@ -779,10 +790,6 @@ def IUIAutomationEventHandler_HandleAutomationEvent(self,sender,eventID): f"Ignoring event {NVDAEventName} because ignored by object itself" ) return - if obj==focus: - if _isDebug(): - log.debug("handleAutomationEvent: redirecting event to focus") - obj=focus if _isDebug(): log.debug( f"handleAutomationEvent: queuing NVDA event {NVDAEventName} " From de5332f119ac0b6673df0deb08a0e90ff55fd6d7 Mon Sep 17 00:00:00 2001 From: Michael Curran Date: Mon, 17 Oct 2022 20:06:36 +1000 Subject: [PATCH 03/10] UIA event handlers: pass windowHandle into NvDAObject constructor if we already have it. --- source/UIAHandler/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/UIAHandler/__init__.py b/source/UIAHandler/__init__.py index 6a6ed75d1eb..c747cd2113c 100644 --- a/source/UIAHandler/__init__.py +++ b/source/UIAHandler/__init__.py @@ -764,7 +764,7 @@ def IUIAutomationEventHandler_HandleAutomationEvent(self,sender,eventID): return if not obj: try: - obj = NVDAObjects.UIA.UIA(UIAElement=sender) + obj = NVDAObjects.UIA.UIA(windowHandle=window, UIAElement=sender) except Exception: if _isDebug(): log.debugWarning( @@ -854,7 +854,7 @@ def IUIAutomationFocusChangedEventHandler_HandleFocusChangedEvent(self,sender): ) return try: - obj = NVDAObjects.UIA.UIA(UIAElement=sender) + obj = NVDAObjects.UIA.UIA(windowHandle=window, UIAElement=sender) except Exception: if _isDebug(): log.debugWarning( @@ -950,7 +950,7 @@ def IUIAutomationPropertyChangedEventHandler_HandlePropertyChangedEvent(self,sen return if not obj: try: - obj = NVDAObjects.UIA.UIA(UIAElement=sender) + obj = NVDAObjects.UIA.UIA(windowHandle=window, UIAElement=sender) except Exception: if _isDebug(): log.debugWarning( From 9b000206fe60b31565f5ad110115cd723ef5e5e0 Mon Sep 17 00:00:00 2001 From: Michael Curran Date: Sun, 23 Apr 2023 22:57:14 +1000 Subject: [PATCH 04/10] Implement a new UI Automation event handler COM object in c++ which avoids calling into Python, and instead records the events, and requests NvDA's main thread to later wake and read off the events when it can. This COM object also coalesces duplicate automation and propertyChange events for the same element, so that NVDA is no longer flooded with duplicate events, E.g. textChange events in Windows consoles. --- nvdaHelper/local/UIAEventLimiter/api.cpp | 30 +++ .../local/UIAEventLimiter/eventRecord.h | 62 +++++ .../rateLimitedEventHandler.cpp | 213 ++++++++++++++++++ .../UIAEventLimiter/rateLimitedEventHandler.h | 50 ++++ nvdaHelper/local/UIAEventLimiter/utils.cpp | 29 +++ nvdaHelper/local/UIAEventLimiter/utils.h | 7 + nvdaHelper/local/nvdaHelperLocal.def | 2 + nvdaHelper/local/sconscript | 3 + source/UIAHandler/__init__.py | 46 +++- source/core.py | 4 +- 10 files changed, 434 insertions(+), 12 deletions(-) create mode 100644 nvdaHelper/local/UIAEventLimiter/api.cpp create mode 100644 nvdaHelper/local/UIAEventLimiter/eventRecord.h create mode 100644 nvdaHelper/local/UIAEventLimiter/rateLimitedEventHandler.cpp create mode 100644 nvdaHelper/local/UIAEventLimiter/rateLimitedEventHandler.h create mode 100644 nvdaHelper/local/UIAEventLimiter/utils.cpp create mode 100644 nvdaHelper/local/UIAEventLimiter/utils.h diff --git a/nvdaHelper/local/UIAEventLimiter/api.cpp b/nvdaHelper/local/UIAEventLimiter/api.cpp new file mode 100644 index 00000000000..544dcbec5f2 --- /dev/null +++ b/nvdaHelper/local/UIAEventLimiter/api.cpp @@ -0,0 +1,30 @@ +#include +#include +#include "rateLimitedEventHandler.h" + +HRESULT rateLimitedUIAEventHandler_create(IUnknown* pExistingHandler, HWND messageWindow, UINT flushMessage, RateLimitedEventHandler** ppRateLimitedEventHandler) { + LOG_DEBUG(L"rateLimitedUIAEventHandler_create called"); + if(!pExistingHandler || !messageWindow || !ppRateLimitedEventHandler) { + LOG_ERROR(L"rateLimitedUIAEventHandler_create: one or more NULL arguments"); + return E_INVALIDARG; + } + + // Create the RateLimitedEventHandler instance + *ppRateLimitedEventHandler = new RateLimitedEventHandler(pExistingHandler, messageWindow, flushMessage); + if (!(*ppRateLimitedEventHandler)) { + LOG_ERROR(L"rateLimitedUIAEventHandler_create: Could not create RateLimitedUIAEventHandler. Returning"); + return E_OUTOFMEMORY; + } + LOG_DEBUG(L"rateLimitedUIAEventHandler_create: done"); + return S_OK; +} + +HRESULT rateLimitedUIAEventHandler_flush(RateLimitedEventHandler* pRateLimitedEventHandler) { + LOG_DEBUG(L"rateLimitedUIAEventHandler_flush called"); + if(!pRateLimitedEventHandler) { + LOG_ERROR(L"rateLimitedUIAEventHandler_flush: invalid argument. Returning"); + return E_INVALIDARG; + } + pRateLimitedEventHandler->flush(); + return S_OK; +} diff --git a/nvdaHelper/local/UIAEventLimiter/eventRecord.h b/nvdaHelper/local/UIAEventLimiter/eventRecord.h new file mode 100644 index 00000000000..e542a188822 --- /dev/null +++ b/nvdaHelper/local/UIAEventLimiter/eventRecord.h @@ -0,0 +1,62 @@ +#pragma once + +#include +#include +#include +#include +#include "utils.h" + +struct AutomationEventRecord_t { + static const bool isCoalesceable = true; + CComPtr sender; + EVENTID eventID; + std::vector generateCoalescingKey() const { + auto key = getRuntimeIDFromElement(sender); + key.push_back(eventID); + return key; + } +}; + +struct PropertyChangedEventRecord_t { + static const bool isCoalesceable = true; + CComPtr sender; + PROPERTYID propertyID; + CComVariant newValue; + std::vector generateCoalescingKey() const { + auto key = getRuntimeIDFromElement(sender); + key.push_back(UIA_AutomationPropertyChangedEventId); + key.push_back(propertyID); + return key; + } +}; + +struct FocusChangedEventRecord_t { + static const bool isCoalesceable = false; + CComPtr sender; +}; + +struct NotificationEventRecord_t { + static const bool isCoalesceable = false; + CComPtr sender; + NotificationKind notificationKind; + NotificationProcessing notificationProcessing; + CComBSTR displayString; + CComBSTR activityID; +}; + +struct ActiveTextPositionChangedEventRecord_t { + static const bool isCoalesceable = false; + CComPtr sender; + CComPtr range; +}; + +using AnyEventRecord_t = std::variant; + +template +concept EventRecordConstraints = requires(T t) { + { t.isCoalesceable } -> std::same_as; + requires (T::isCoalesceable == false) || requires(T t) { + { t.generateCoalescingKey() } -> std::same_as>; + }; + { AnyEventRecord_t(t)} -> std::same_as; +}; diff --git a/nvdaHelper/local/UIAEventLimiter/rateLimitedEventHandler.cpp b/nvdaHelper/local/UIAEventLimiter/rateLimitedEventHandler.cpp new file mode 100644 index 00000000000..1ef126694f4 --- /dev/null +++ b/nvdaHelper/local/UIAEventLimiter/rateLimitedEventHandler.cpp @@ -0,0 +1,213 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "eventRecord.h" +#include "rateLimitedEventHandler.h" + +template +HRESULT RateLimitedEventHandler::queueEvent(EventRecordArgTypes&&... args) { + LOG_DEBUG(L"RateLimitedUIAEventHandler::queueEvent called"); + bool needsFlush = false; + const unsigned int flushTimeMS = (EventRecordClass::isCoalesceable)?30:0; + { std::lock_guard lock(mtx); + if(!EventRecordClass::isCoalesceable || m_eventRecords.empty()) { + needsFlush = true; + } + LOG_DEBUG(L"RateLimitedUIAEventHandler::queueEvent: Inserting new event"); + auto& recordVar = m_eventRecords.emplace_back(std::in_place_type_t{}, args...); + auto recordVarIter = m_eventRecords.end(); + recordVarIter--; + auto& record = std::get(recordVar); + if constexpr(EventRecordClass::isCoalesceable) { + LOG_DEBUG(L"RateLimitedUIAEventHandler::queueEvent: Is a coalesceable event"); + auto coalescingKey = record.generateCoalescingKey(); + auto existingKeyIter = m_eventRecordsByKey.find(coalescingKey); + if(existingKeyIter != m_eventRecordsByKey.end()) { + LOG_DEBUG(L"RateLimitedUIAEventHandler::queueEvent: found existing event with same key"); + auto& [existingRecordVarIter,existingCoalesceCount] = existingKeyIter->second; + LOG_DEBUG(L"RateLimitedUIAEventHandler::queueEvent: updating key and count to "<<(existingCoalesceCount+1)); + existingKeyIter->second = {recordVarIter, existingCoalesceCount + 1}; + LOG_DEBUG(L"RateLimitedUIAEventHandler::queueEvent: erasing old item"); + m_eventRecords.erase(existingRecordVarIter); + } else { + LOG_DEBUG(L"RateLimitedUIAEventHandler::queueEvent: Adding key"); + m_eventRecordsByKey.insert_or_assign(coalescingKey, std::pair(recordVarIter, 1)); + } + } + } + if(needsFlush) { + LOG_DEBUG(L"RateLimitedUIAEventHandler::queueEvent: posting flush message"); + PostMessage(m_messageWindow, m_flushMessage, reinterpret_cast(this), flushTimeMS); + } + return S_OK; +} + +HRESULT RateLimitedEventHandler::emitEvent(const AutomationEventRecord_t& record) const { + LOG_DEBUG(L"RateLimitedUIAEventHandler::emitAutomationEvent called"); + if(!m_pExistingAutomationEventHandler) { + LOG_ERROR(L"RateLimitedUIAEventHandler::emitAutomationEvent: interface not supported."); + return E_NOINTERFACE; + } + LOG_DEBUG(L"Emitting automationEvent for eventID "<HandleAutomationEvent(record.sender, record.eventID); +} + +HRESULT RateLimitedEventHandler::emitEvent(const FocusChangedEventRecord_t& record) const { + LOG_DEBUG(L"RateLimitedUIAEventHandler::emitFocusChangedEvent called"); + if(!m_pExistingFocusChangedEventHandler) { + LOG_ERROR(L"RateLimitedUIAEventHandler::emitFocusChangedEvent: interface not supported."); + return E_NOINTERFACE; + } + return m_pExistingFocusChangedEventHandler->HandleFocusChangedEvent(record.sender); +} + +HRESULT RateLimitedEventHandler::emitEvent(const PropertyChangedEventRecord_t& record) const { + LOG_DEBUG(L"RateLimitedUIAEventHandler::emitPropertyChangedEvent called"); + if(!m_pExistingPropertyChangedEventHandler) { + LOG_ERROR(L"RateLimitedUIAEventHandler::emitPropertyChangedEvent: interface not supported."); + return E_NOINTERFACE; + } + return m_pExistingPropertyChangedEventHandler->HandlePropertyChangedEvent(record.sender, record.propertyID, record.newValue); +} + +HRESULT RateLimitedEventHandler::emitEvent(const NotificationEventRecord_t& record) const { + LOG_DEBUG(L"RateLimitedUIAEventHandler::emitNotificationEvent called"); + if(!m_pExistingNotificationEventHandler) { + LOG_ERROR(L"RateLimitedUIAEventHandler::emitNotificationChangedEvent: interface not supported."); + return E_NOINTERFACE; + } + return m_pExistingNotificationEventHandler->HandleNotificationEvent(record.sender, record.notificationKind, record.notificationProcessing, record.displayString, record.activityID); +} + +HRESULT RateLimitedEventHandler::emitEvent(const ActiveTextPositionChangedEventRecord_t& record) const { + LOG_DEBUG(L"RateLimitedUIAEventHandler::emitActiveTextPositionChangedEvent called"); + if(!m_pExistingActiveTextPositionChangedEventHandler) { + LOG_ERROR(L"RateLimitedUIAEventHandler::emitActiveTextPositionChangedEvent: interface not supported."); + return E_NOINTERFACE; + } + return m_pExistingActiveTextPositionChangedEventHandler->HandleActiveTextPositionChangedEvent(record.sender, record.range); +} + +RateLimitedEventHandler::~RateLimitedEventHandler() { + LOG_DEBUG(L"RateLimitedUIAEventHandler::~RateLimitedUIAEventHandler called"); +} + +RateLimitedEventHandler::RateLimitedEventHandler(IUnknown* pExistingHandler, HWND messageWindow, UINT flushMessage) + : m_messageWindow(messageWindow), m_flushMessage(flushMessage), m_refCount(1), m_pExistingAutomationEventHandler(pExistingHandler), m_pExistingFocusChangedEventHandler(pExistingHandler), m_pExistingPropertyChangedEventHandler(pExistingHandler), m_pExistingNotificationEventHandler(pExistingHandler), m_pExistingActiveTextPositionChangedEventHandler(pExistingHandler) { + LOG_DEBUG(L"RateLimitedUIAEventHandler::RateLimitedUIAEventHandler called"); +} + +// IUnknown methods +ULONG STDMETHODCALLTYPE RateLimitedEventHandler::AddRef() { + return InterlockedIncrement(&m_refCount); +} + +ULONG STDMETHODCALLTYPE RateLimitedEventHandler::Release() { + ULONG refCount = InterlockedDecrement(&m_refCount); + if (refCount == 0) { + delete this; + } + return refCount; +} + +HRESULT STDMETHODCALLTYPE RateLimitedEventHandler::QueryInterface(REFIID riid, void** ppInterface) { + if (riid == __uuidof(IUnknown)) { + *ppInterface = static_cast(this); + AddRef(); + return S_OK; + } else if (riid == __uuidof(IUIAutomationEventHandler)) { + *ppInterface = static_cast(this); + AddRef(); + return S_OK; + } else if (riid == __uuidof(IUIAutomationFocusChangedEventHandler)) { + *ppInterface = static_cast(this); + AddRef(); + return S_OK; + } else if (riid == __uuidof(IUIAutomationPropertyChangedEventHandler)) { + *ppInterface = static_cast(this); + AddRef(); + return S_OK; + } else if (riid == __uuidof(IUIAutomationNotificationEventHandler)) { + *ppInterface = static_cast(this); + AddRef(); + return S_OK; + } else if (riid == __uuidof(IUIAutomationActiveTextPositionChangedEventHandler)) { + *ppInterface = static_cast(this); + AddRef(); + return S_OK; + } + *ppInterface = nullptr; + return E_NOINTERFACE; +} + +// IUIAutomationEventHandler method +HRESULT STDMETHODCALLTYPE RateLimitedEventHandler::HandleAutomationEvent(IUIAutomationElement* sender, EVENTID eventID) { + LOG_DEBUG(L"RateLimitedUIAEventHandler::HandleAutomationEvent called"); + LOG_DEBUG(L"Queuing automationEvent for eventID "<(sender, eventID); +} + +// IUIAutomationFocusEventHandler method +HRESULT STDMETHODCALLTYPE RateLimitedEventHandler::HandleFocusChangedEvent(IUIAutomationElement* sender) { + LOG_DEBUG(L"RateLimitedUIAEventHandler::HandleFocusChangedEvent called"); + return queueEvent(sender); +} + +// IUIAutomationPropertyChangedEventHandler method +HRESULT STDMETHODCALLTYPE RateLimitedEventHandler::HandlePropertyChangedEvent(IUIAutomationElement* sender, PROPERTYID propertyID, VARIANT newValue) { + LOG_DEBUG(L"RateLimitedUIAEventHandler::HandlePropertyChangedEvent called"); + return queueEvent(sender, propertyID, newValue); +} + +// IUIAutomationNotificationEventHandler method +HRESULT STDMETHODCALLTYPE RateLimitedEventHandler::HandleNotificationEvent(IUIAutomationElement* sender, NotificationKind notificationKind, NotificationProcessing notificationProcessing, BSTR displayString, BSTR activityID) { + LOG_DEBUG(L"RateLimitedUIAEventHandler::HandleNotificationEvent called"); + return queueEvent(sender, notificationKind, notificationProcessing, displayString, activityID); +} + +// IUIAutomationActiveTextPositionchangedEventHandler method +HRESULT STDMETHODCALLTYPE RateLimitedEventHandler::HandleActiveTextPositionChangedEvent(IUIAutomationElement* sender, IUIAutomationTextRange* range) { + LOG_DEBUG(L"RateLimitedUIAEventHandler::HandleActiveTextPositionChangedEvent called"); + return queueEvent(sender, range); +} + +void RateLimitedEventHandler::flush() { + LOG_DEBUG(L"RateLimitedUIAEventHandler::flush called"); + decltype(m_eventRecords) eventRecordsCopy; + decltype(m_eventRecordsByKey) eventRecordsByKeyCopy; + { std::lock_guard lock(mtx); + eventRecordsCopy.swap(m_eventRecords); + eventRecordsByKeyCopy.swap(m_eventRecordsByKey); + } + + // Emit events + LOG_DEBUG(L"RateLimitedUIAEventHandler::flush: Emitting events..."); + for(const auto& recordVar: eventRecordsCopy) { + std::visit([this](const auto& record) { + this->emitEvent(record); + }, recordVar); + } + /* + unsigned int count = std::accumulate(eventRecordsByKeyCopy.begin(), eventRecordsByKeyCopy.end(), 0, [](const auto& acc, const auto& i) { + auto count = i.second.second; + if(count > 1) { + return acc + count; + } + return acc; + }); + if(count > 0) { + Beep(440 + (count*10), 40); + } + */ + LOG_DEBUG(L"RateLimitedUIAEventHandler::flush: done emitting events"); + +} diff --git a/nvdaHelper/local/UIAEventLimiter/rateLimitedEventHandler.h b/nvdaHelper/local/UIAEventLimiter/rateLimitedEventHandler.h new file mode 100644 index 00000000000..d7ed4996a83 --- /dev/null +++ b/nvdaHelper/local/UIAEventLimiter/rateLimitedEventHandler.h @@ -0,0 +1,50 @@ +#include +#include +#include +#include +#include +#include "eventRecord.h" + +class RateLimitedEventHandler: public IUIAutomationEventHandler, public IUIAutomationFocusChangedEventHandler, public IUIAutomationPropertyChangedEventHandler, public IUIAutomationNotificationEventHandler, public IUIAutomationActiveTextPositionChangedEventHandler { +private: + unsigned long m_refCount; + CComQIPtr m_pExistingAutomationEventHandler; + CComQIPtr m_pExistingFocusChangedEventHandler; + CComQIPtr m_pExistingPropertyChangedEventHandler; + CComQIPtr m_pExistingNotificationEventHandler; + CComQIPtr m_pExistingActiveTextPositionChangedEventHandler; + HWND m_messageWindow; + UINT m_flushMessage; + std::mutex mtx; + std::list m_eventRecords; + std::map, std::pair> m_eventRecordsByKey; + + template HRESULT queueEvent(EventRecordArgTypes&&... args); + + HRESULT emitEvent(const AutomationEventRecord_t& record) const; + HRESULT emitEvent(const FocusChangedEventRecord_t& record) const; + HRESULT emitEvent(const PropertyChangedEventRecord_t& record) const; + HRESULT emitEvent(const NotificationEventRecord_t& record) const; + HRESULT emitEvent(const ActiveTextPositionChangedEventRecord_t& record) const; + + ~RateLimitedEventHandler(); + + public: + + RateLimitedEventHandler(IUnknown* pExistingHandler, HWND messageWindow, UINT flushMessage); + + // IUnknown methods + ULONG STDMETHODCALLTYPE AddRef(); + ULONG STDMETHODCALLTYPE Release(); + HRESULT STDMETHODCALLTYPE QueryInterface(REFIID riid, void** ppInterface); + + // IUIAutomationEventHandler methods + HRESULT STDMETHODCALLTYPE HandleAutomationEvent(IUIAutomationElement* sender, EVENTID eventID); + HRESULT STDMETHODCALLTYPE HandleFocusChangedEvent(IUIAutomationElement* sender); + HRESULT STDMETHODCALLTYPE HandlePropertyChangedEvent(IUIAutomationElement* sender, PROPERTYID propertyID, VARIANT newValue); + HRESULT STDMETHODCALLTYPE HandleNotificationEvent(IUIAutomationElement* sender, NotificationKind notificationKind, NotificationProcessing notificationProcessing, BSTR displayString, BSTR activityID); + HRESULT STDMETHODCALLTYPE HandleActiveTextPositionChangedEvent(IUIAutomationElement* sender, IUIAutomationTextRange* range); + + void flush(); + +}; diff --git a/nvdaHelper/local/UIAEventLimiter/utils.cpp b/nvdaHelper/local/UIAEventLimiter/utils.cpp new file mode 100644 index 00000000000..16934ac874b --- /dev/null +++ b/nvdaHelper/local/UIAEventLimiter/utils.cpp @@ -0,0 +1,29 @@ +#include +#include +#include +#include "utils.h" + +std::vector SafeArrayToVector(SAFEARRAY* pSafeArray) { + std::vector vec; + int* data; + HRESULT hr = SafeArrayAccessData(pSafeArray, (void**)&data); + if (SUCCEEDED(hr)) { + LONG lowerBound, upperBound; + SafeArrayGetLBound(pSafeArray, 1, &lowerBound); + SafeArrayGetUBound(pSafeArray, 1, &upperBound); + vec.assign(data, data + (upperBound - lowerBound + 1)); + SafeArrayUnaccessData(pSafeArray); + } + return vec; +} + +std::vector getRuntimeIDFromElement(IUIAutomationElement* pElement) { + SAFEARRAY* runtimeIdArray; + HRESULT hr = pElement->GetRuntimeId(&runtimeIdArray); + if (FAILED(hr)) { + return {}; + } + std::vector runtimeID = SafeArrayToVector(runtimeIdArray); + SafeArrayDestroy(runtimeIdArray); + return runtimeID; +} diff --git a/nvdaHelper/local/UIAEventLimiter/utils.h b/nvdaHelper/local/UIAEventLimiter/utils.h new file mode 100644 index 00000000000..44d1b917ec8 --- /dev/null +++ b/nvdaHelper/local/UIAEventLimiter/utils.h @@ -0,0 +1,7 @@ +#include +#include +#include + +std::vector SafeArrayToVector(SAFEARRAY* pSafeArray); + +std::vector getRuntimeIDFromElement(IUIAutomationElement* pElement); diff --git a/nvdaHelper/local/nvdaHelperLocal.def b/nvdaHelper/local/nvdaHelperLocal.def index ee234a84523..39d6fc8d0e4 100644 --- a/nvdaHelper/local/nvdaHelperLocal.def +++ b/nvdaHelper/local/nvdaHelperLocal.def @@ -67,3 +67,5 @@ EXPORTS getOleClipboardText _nvdaControllerInternal_reportLiveRegion _nvdaControllerInternal_openConfigDirectory + rateLimitedUIAEventHandler_create + rateLimitedUIAEventHandler_flush diff --git a/nvdaHelper/local/sconscript b/nvdaHelper/local/sconscript index 3c29b37b6ad..8db27a1495e 100644 --- a/nvdaHelper/local/sconscript +++ b/nvdaHelper/local/sconscript @@ -89,6 +89,9 @@ localLib=env.SharedLibrary( "nvdaHelperLocal.def", "textUtils.cpp", "UIAUtils.cpp", + "UIAEventLimiter/api.cpp", + "UIAEventLimiter/rateLimitedEventHandler.cpp", + "UIAEventLimiter/utils.cpp", "mixer.cpp", "oleUtils.cpp", ], diff --git a/source/UIAHandler/__init__.py b/source/UIAHandler/__init__.py index c747cd2113c..1022834c29f 100644 --- a/source/UIAHandler/__init__.py +++ b/source/UIAHandler/__init__.py @@ -9,6 +9,9 @@ from ctypes import ( oledll, windll, + POINTER, + CFUNCTYPE, + c_voidp, ) import comtypes.client @@ -19,10 +22,12 @@ byref, CLSCTX_INPROC_SERVER, CoCreateInstance, + IUnknown, ) import threading import time +import winAPI.messageWindow import IAccessibleHandler.internalWinEventHandler import config from config import ( @@ -46,6 +51,8 @@ from typing import Dict from queue import Queue import aria +import core +import NVDAHelper from . import remote as UIARemote @@ -507,6 +514,14 @@ def MTAThreadFunc(self): self.rootElement=self.clientObject.getRootElementBuildCache(self.baseCacheRequest) self.reservedNotSupportedValue=self.clientObject.ReservedNotSupportedValue self.ReservedMixedAttributeValue=self.clientObject.ReservedMixedAttributeValue + self.pRateLimitedEventHandler = POINTER(IUnknown)() + NVDAHelper.localLib.rateLimitedUIAEventHandler_create( + self._com_pointers_[IUnknown._iid_], + core.messageWindow.handle, + WM_NVDA_UIA_FLUSH, + byref(self.pRateLimitedEventHandler) + ) + winAPI.messageWindow.pre_handleWindowMessage.register(eventLimiterWindowProc) if utils._shouldSelectivelyRegister(): self._createLocalEventHandlerGroup() self._registerGlobalEventHandlers() @@ -528,7 +543,7 @@ def MTAThreadFunc(self): self.clientObject.RemoveAllEventHandlers() def _registerGlobalEventHandlers(self): - self.clientObject.AddFocusChangedEventHandler(self.baseCacheRequest, self) + self.clientObject.AddFocusChangedEventHandler(self.baseCacheRequest, self.pRateLimitedEventHandler) if isinstance(self.clientObject, UIA.IUIAutomation6): self.globalEventHandlerGroup = self.clientObject.CreateEventHandlerGroup() else: @@ -536,7 +551,7 @@ def _registerGlobalEventHandlers(self): self.globalEventHandlerGroup.AddPropertyChangedEventHandler( UIA.TreeScope_Subtree, self.baseCacheRequest, - self, + self.pRateLimitedEventHandler, *self.clientObject.IntSafeArrayToNativeArray( globalEventHandlerGroupUIAPropertyIds if utils._shouldSelectivelyRegister() @@ -552,7 +567,7 @@ def _registerGlobalEventHandlers(self): eventId, UIA.TreeScope_Subtree, self.baseCacheRequest, - self + self.pRateLimitedEventHandler ) if ( not utils._shouldSelectivelyRegister() @@ -563,20 +578,20 @@ def _registerGlobalEventHandlers(self): UIA.UIA_Text_TextChangedEventId, UIA.TreeScope_Subtree, self.baseCacheRequest, - self + self.pRateLimitedEventHandler ) # #7984: add support for notification event (IUIAutomation5, part of Windows 10 build 16299 and later). if isinstance(self.clientObject, UIA.IUIAutomation5): self.globalEventHandlerGroup.AddNotificationEventHandler( UIA.TreeScope_Subtree, self.baseCacheRequest, - self + self.pRateLimitedEventHandler ) if isinstance(self.clientObject, UIA.IUIAutomation6): self.globalEventHandlerGroup.AddActiveTextPositionChangedEventHandler( UIA.TreeScope_Subtree, self.baseCacheRequest, - self + self.pRateLimitedEventHandler ) self.addEventHandlerGroup(self.rootElement, self.globalEventHandlerGroup) @@ -590,13 +605,13 @@ def _createLocalEventHandlerGroup(self): self.localEventHandlerGroup.AddPropertyChangedEventHandler( UIA.TreeScope_Ancestors | UIA.TreeScope_Element, self.baseCacheRequest, - self, + self.pRateLimitedEventHandler, *self.clientObject.IntSafeArrayToNativeArray(localEventHandlerGroupUIAPropertyIds) ) self.localEventHandlerGroupWithTextChanges.AddPropertyChangedEventHandler( UIA.TreeScope_Ancestors | UIA.TreeScope_Element, self.baseCacheRequest, - self, + self.pRateLimitedEventHandler, *self.clientObject.IntSafeArrayToNativeArray(localEventHandlerGroupUIAPropertyIds) ) for eventId in localEventHandlerGroupUIAEventIds: @@ -604,19 +619,19 @@ def _createLocalEventHandlerGroup(self): eventId, UIA.TreeScope_Ancestors | UIA.TreeScope_Element, self.baseCacheRequest, - self + self.pRateLimitedEventHandler ) self.localEventHandlerGroupWithTextChanges.AddAutomationEventHandler( eventId, UIA.TreeScope_Ancestors | UIA.TreeScope_Element, self.baseCacheRequest, - self + self.pRateLimitedEventHandler ) self.localEventHandlerGroupWithTextChanges.AddAutomationEventHandler( UIA.UIA_Text_TextChangedEventId, UIA.TreeScope_Ancestors | UIA.TreeScope_Element, self.baseCacheRequest, - self + self.pRateLimitedEventHandler ) def addEventHandlerGroup(self, element, eventHandlerGroup): @@ -1390,3 +1405,12 @@ def terminate(): def _isDebug(): return config.conf["debugLog"]["UIA"] + + +WM_NVDA_UIA_FLUSH = winUser.registerWindowMessage("WM_NVDA_UIA_FLUSH") +def eventLimiterWindowProc(msg, wParam, lParam): + if msg == WM_NVDA_UIA_FLUSH: + if lParam == 0: + NVDAHelper.localLib.rateLimitedUIAEventHandler_flush(wParam) + else: + core.callLater(lParam, NVDAHelper.localLib.rateLimitedUIAEventHandler_flush, wParam) diff --git a/source/core.py b/source/core.py index bb2bbe7a755..83f9599da18 100644 --- a/source/core.py +++ b/source/core.py @@ -477,6 +477,8 @@ def _doLoseFocus(): except Exception: log.exception("Lose focus error") +messageWindow = None + def main(): """NVDA's core main loop. @@ -619,8 +621,8 @@ def onEndSession(evt): from winAPI.messageWindow import _MessageWindow import versionInfo + global messageWindow messageWindow = _MessageWindow(versionInfo.name) - # initialize wxpython localization support wxLocaleObj = wx.Locale() wxLang = getWxLangOrNone() From 360552431ba26beaba4485dedb4dc6db6a3a32e1 Mon Sep 17 00:00:00 2001 From: Michael Curran Date: Mon, 1 May 2023 10:03:07 +1000 Subject: [PATCH 05/10] Add lisence and fix linting --- nvdaHelper/local/UIAEventLimiter/api.cpp | 14 ++++++++++++++ nvdaHelper/local/UIAEventLimiter/eventRecord.h | 14 ++++++++++++++ .../UIAEventLimiter/rateLimitedEventHandler.cpp | 14 ++++++++++++++ .../UIAEventLimiter/rateLimitedEventHandler.h | 14 ++++++++++++++ nvdaHelper/local/UIAEventLimiter/utils.cpp | 14 ++++++++++++++ nvdaHelper/local/UIAEventLimiter/utils.h | 14 ++++++++++++++ source/UIAHandler/__init__.py | 2 ++ source/core.py | 1 + 8 files changed, 87 insertions(+) diff --git a/nvdaHelper/local/UIAEventLimiter/api.cpp b/nvdaHelper/local/UIAEventLimiter/api.cpp index 544dcbec5f2..73ec92439be 100644 --- a/nvdaHelper/local/UIAEventLimiter/api.cpp +++ b/nvdaHelper/local/UIAEventLimiter/api.cpp @@ -1,3 +1,17 @@ +/* +This file is a part of the NVDA project. +URL: http://github.com/nvaccess/nvda/ +Copyright 2023 NV Access Limited. + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License version 2.0, as published by + the Free Software Foundation. + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +This license can be found at: +http://www.gnu.org/licenses/old-licenses/gpl-2.0.html +*/ + #include #include #include "rateLimitedEventHandler.h" diff --git a/nvdaHelper/local/UIAEventLimiter/eventRecord.h b/nvdaHelper/local/UIAEventLimiter/eventRecord.h index e542a188822..55c00ad3b48 100644 --- a/nvdaHelper/local/UIAEventLimiter/eventRecord.h +++ b/nvdaHelper/local/UIAEventLimiter/eventRecord.h @@ -1,3 +1,17 @@ +/* +This file is a part of the NVDA project. +URL: http://github.com/nvaccess/nvda/ +Copyright 2023 NV Access Limited. + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License version 2.0, as published by + the Free Software Foundation. + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +This license can be found at: +http://www.gnu.org/licenses/old-licenses/gpl-2.0.html +*/ + #pragma once #include diff --git a/nvdaHelper/local/UIAEventLimiter/rateLimitedEventHandler.cpp b/nvdaHelper/local/UIAEventLimiter/rateLimitedEventHandler.cpp index 1ef126694f4..8d8816a4380 100644 --- a/nvdaHelper/local/UIAEventLimiter/rateLimitedEventHandler.cpp +++ b/nvdaHelper/local/UIAEventLimiter/rateLimitedEventHandler.cpp @@ -1,3 +1,17 @@ +/* +This file is a part of the NVDA project. +URL: http://github.com/nvaccess/nvda/ +Copyright 2023 NV Access Limited. + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License version 2.0, as published by + the Free Software Foundation. + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +This license can be found at: +http://www.gnu.org/licenses/old-licenses/gpl-2.0.html +*/ + #include #include #include diff --git a/nvdaHelper/local/UIAEventLimiter/rateLimitedEventHandler.h b/nvdaHelper/local/UIAEventLimiter/rateLimitedEventHandler.h index d7ed4996a83..56537a65c8f 100644 --- a/nvdaHelper/local/UIAEventLimiter/rateLimitedEventHandler.h +++ b/nvdaHelper/local/UIAEventLimiter/rateLimitedEventHandler.h @@ -1,3 +1,17 @@ +/* +This file is a part of the NVDA project. +URL: http://github.com/nvaccess/nvda/ +Copyright 2023 NV Access Limited. + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License version 2.0, as published by + the Free Software Foundation. + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +This license can be found at: +http://www.gnu.org/licenses/old-licenses/gpl-2.0.html +*/ + #include #include #include diff --git a/nvdaHelper/local/UIAEventLimiter/utils.cpp b/nvdaHelper/local/UIAEventLimiter/utils.cpp index 16934ac874b..ae43af60fa1 100644 --- a/nvdaHelper/local/UIAEventLimiter/utils.cpp +++ b/nvdaHelper/local/UIAEventLimiter/utils.cpp @@ -1,3 +1,17 @@ +/* +This file is a part of the NVDA project. +URL: http://github.com/nvaccess/nvda/ +Copyright 2023 NV Access Limited. + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License version 2.0, as published by + the Free Software Foundation. + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +This license can be found at: +http://www.gnu.org/licenses/old-licenses/gpl-2.0.html +*/ + #include #include #include diff --git a/nvdaHelper/local/UIAEventLimiter/utils.h b/nvdaHelper/local/UIAEventLimiter/utils.h index 44d1b917ec8..cbf4cb5d53c 100644 --- a/nvdaHelper/local/UIAEventLimiter/utils.h +++ b/nvdaHelper/local/UIAEventLimiter/utils.h @@ -1,3 +1,17 @@ +/* +This file is a part of the NVDA project. +URL: http://github.com/nvaccess/nvda/ +Copyright 2023 NV Access Limited. + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License version 2.0, as published by + the Free Software Foundation. + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +This license can be found at: +http://www.gnu.org/licenses/old-licenses/gpl-2.0.html +*/ + #include #include #include diff --git a/source/UIAHandler/__init__.py b/source/UIAHandler/__init__.py index 1022834c29f..0057ec30492 100644 --- a/source/UIAHandler/__init__.py +++ b/source/UIAHandler/__init__.py @@ -1408,6 +1408,8 @@ def _isDebug(): WM_NVDA_UIA_FLUSH = winUser.registerWindowMessage("WM_NVDA_UIA_FLUSH") + + def eventLimiterWindowProc(msg, wParam, lParam): if msg == WM_NVDA_UIA_FLUSH: if lParam == 0: diff --git a/source/core.py b/source/core.py index 83f9599da18..f3150c3dcb3 100644 --- a/source/core.py +++ b/source/core.py @@ -477,6 +477,7 @@ def _doLoseFocus(): except Exception: log.exception("Lose focus error") + messageWindow = None From d191e723e62002d2790bfdc4a8b4bd57da0d0fd0 Mon Sep 17 00:00:00 2001 From: Michael Curran Date: Tue, 2 May 2023 08:45:54 +1000 Subject: [PATCH 06/10] UIAEventLimiter: improve EventRecordContaint c++20 concept so as to not crash VS Code Intellisense. --- .../local/UIAEventLimiter/eventRecord.h | 7 ++++-- .../UIAEventLimiter/rateLimitedEventHandler.h | 2 +- nvdaHelper/local/UIAEventLimiter/utils.h | 24 ++++++++++++++----- 3 files changed, 24 insertions(+), 9 deletions(-) diff --git a/nvdaHelper/local/UIAEventLimiter/eventRecord.h b/nvdaHelper/local/UIAEventLimiter/eventRecord.h index 55c00ad3b48..b0f4d69ad1d 100644 --- a/nvdaHelper/local/UIAEventLimiter/eventRecord.h +++ b/nvdaHelper/local/UIAEventLimiter/eventRecord.h @@ -64,13 +64,16 @@ struct ActiveTextPositionChangedEventRecord_t { CComPtr range; }; -using AnyEventRecord_t = std::variant; +using EventRecordVariant_t = std::variant; template concept EventRecordConstraints = requires(T t) { + // type must have a static const bool member called 'isCoalesceable' { t.isCoalesceable } -> std::same_as; + // if 'isCoaleseable' is true, then the type must have a 'generateCoalescingKey' method. requires (T::isCoalesceable == false) || requires(T t) { { t.generateCoalescingKey() } -> std::same_as>; }; - { AnyEventRecord_t(t)} -> std::same_as; + // The type must be supported by the EventRecordVariant_t variant type + requires supports_alternative; }; diff --git a/nvdaHelper/local/UIAEventLimiter/rateLimitedEventHandler.h b/nvdaHelper/local/UIAEventLimiter/rateLimitedEventHandler.h index 56537a65c8f..f085a5db3c0 100644 --- a/nvdaHelper/local/UIAEventLimiter/rateLimitedEventHandler.h +++ b/nvdaHelper/local/UIAEventLimiter/rateLimitedEventHandler.h @@ -30,7 +30,7 @@ class RateLimitedEventHandler: public IUIAutomationEventHandler, public IUIAutom HWND m_messageWindow; UINT m_flushMessage; std::mutex mtx; - std::list m_eventRecords; + std::list m_eventRecords; std::map, std::pair> m_eventRecordsByKey; template HRESULT queueEvent(EventRecordArgTypes&&... args); diff --git a/nvdaHelper/local/UIAEventLimiter/utils.h b/nvdaHelper/local/UIAEventLimiter/utils.h index cbf4cb5d53c..2a11ef89ef3 100644 --- a/nvdaHelper/local/UIAEventLimiter/utils.h +++ b/nvdaHelper/local/UIAEventLimiter/utils.h @@ -2,20 +2,32 @@ This file is a part of the NVDA project. URL: http://github.com/nvaccess/nvda/ Copyright 2023 NV Access Limited. - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License version 2.0, as published by - the Free Software Foundation. - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License version 2.0, as published by + the Free Software Foundation. + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. This license can be found at: http://www.gnu.org/licenses/old-licenses/gpl-2.0.html */ +#include #include +#include +#include +#include #include #include std::vector SafeArrayToVector(SAFEARRAY* pSafeArray); std::vector getRuntimeIDFromElement(IUIAutomationElement* pElement); + +template +constexpr bool supports_alternative_impl(std::index_sequence) { + return (std::same_as> || ...); +} + +template +concept supports_alternative = supports_alternative_impl(std::make_index_sequence>{}); From 7f1028cd514387eecbe118c2848011bcf1916ec3 Mon Sep 17 00:00:00 2001 From: Michael Curran Date: Tue, 2 May 2023 19:15:23 +1000 Subject: [PATCH 07/10] UIA event limiter: no longer coalesce propertyChange events, as it is better that these are quick for responce, and it is really autoamtion events like textchange which truly need coalescing. --- nvdaHelper/local/UIAEventLimiter/eventRecord.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nvdaHelper/local/UIAEventLimiter/eventRecord.h b/nvdaHelper/local/UIAEventLimiter/eventRecord.h index b0f4d69ad1d..42aa7f18c9a 100644 --- a/nvdaHelper/local/UIAEventLimiter/eventRecord.h +++ b/nvdaHelper/local/UIAEventLimiter/eventRecord.h @@ -32,7 +32,7 @@ struct AutomationEventRecord_t { }; struct PropertyChangedEventRecord_t { - static const bool isCoalesceable = true; + static const bool isCoalesceable = false; CComPtr sender; PROPERTYID propertyID; CComVariant newValue; From 2e268000342518204d9ed4049a0e5e7e53e7408a Mon Sep 17 00:00:00 2001 From: Michael Curran Date: Tue, 2 May 2023 19:16:57 +1000 Subject: [PATCH 08/10] UIA event limiter: ensure not to flood NVDA with needless flush messages after a quick flush is requested and before the flush has actually occured. --- .../rateLimitedEventHandler.cpp | 33 +++++++++++++++---- .../UIAEventLimiter/rateLimitedEventHandler.h | 7 ++++ 2 files changed, 33 insertions(+), 7 deletions(-) diff --git a/nvdaHelper/local/UIAEventLimiter/rateLimitedEventHandler.cpp b/nvdaHelper/local/UIAEventLimiter/rateLimitedEventHandler.cpp index 8d8816a4380..dbc68590af4 100644 --- a/nvdaHelper/local/UIAEventLimiter/rateLimitedEventHandler.cpp +++ b/nvdaHelper/local/UIAEventLimiter/rateLimitedEventHandler.cpp @@ -30,11 +30,30 @@ This license can be found at: template HRESULT RateLimitedEventHandler::queueEvent(EventRecordArgTypes&&... args) { LOG_DEBUG(L"RateLimitedUIAEventHandler::queueEvent called"); - bool needsFlush = false; - const unsigned int flushTimeMS = (EventRecordClass::isCoalesceable)?30:0; + FlushRequest flushRequest = FlushRequest::none; + unsigned int flushTimeMS = 0; { std::lock_guard lock(mtx); - if(!EventRecordClass::isCoalesceable || m_eventRecords.empty()) { - needsFlush = true; + // work out whether we need to request a flush after inserting this event. + if(!EventRecordClass::isCoalesceable) { + // not a coalesceable event so requires a quick flush + // if a quick flush has not been requested already. + if(lastFlushRequest < FlushRequest::quick) { + LOG_DEBUG(L"RateLimitedUIAEventHandler::queueEvent: requesting quick flush after queuing"); + lastFlushRequest = flushRequest = FlushRequest::quick; + } else { + LOG_DEBUG(L"RateLimitedUIAEventHandler::queueEvent: quick flush needed, but quick flush already requested."); + } + } else if(m_eventRecords.empty()) { + // It is a coalesceable event and its the first event since the last flush, + // so requires a delayed flush + // if a delayed or quick flush is not requested already. + if(lastFlushRequest < FlushRequest::delayed) { + LOG_DEBUG(L"RateLimitedUIAEventHandler::queueEvent: requesting delayed flush after queuing"); + lastFlushRequest = flushRequest = FlushRequest::delayed; + flushTimeMS = 30; + } else { + LOG_DEBUG(L"RateLimitedUIAEventHandler::queueEvent: delayed flush needed, but quick flush already requested."); + } } LOG_DEBUG(L"RateLimitedUIAEventHandler::queueEvent: Inserting new event"); auto& recordVar = m_eventRecords.emplace_back(std::in_place_type_t{}, args...); @@ -58,8 +77,8 @@ HRESULT RateLimitedEventHandler::queueEvent(EventRecordArgTypes&&... args) { } } } - if(needsFlush) { - LOG_DEBUG(L"RateLimitedUIAEventHandler::queueEvent: posting flush message"); + if(flushRequest > FlushRequest::none) { + LOG_DEBUG(L"RateLimitedUIAEventHandler::queueEvent: posting flush message with delay of "<(this), flushTimeMS); } return S_OK; @@ -201,6 +220,7 @@ void RateLimitedEventHandler::flush() { { std::lock_guard lock(mtx); eventRecordsCopy.swap(m_eventRecords); eventRecordsByKeyCopy.swap(m_eventRecordsByKey); + lastFlushRequest = FlushRequest::none; } // Emit events @@ -223,5 +243,4 @@ void RateLimitedEventHandler::flush() { } */ LOG_DEBUG(L"RateLimitedUIAEventHandler::flush: done emitting events"); - } diff --git a/nvdaHelper/local/UIAEventLimiter/rateLimitedEventHandler.h b/nvdaHelper/local/UIAEventLimiter/rateLimitedEventHandler.h index f085a5db3c0..4fd9baba6a3 100644 --- a/nvdaHelper/local/UIAEventLimiter/rateLimitedEventHandler.h +++ b/nvdaHelper/local/UIAEventLimiter/rateLimitedEventHandler.h @@ -19,6 +19,12 @@ This license can be found at: #include #include "eventRecord.h" +enum FlushRequest { + none, + delayed, + quick +}; + class RateLimitedEventHandler: public IUIAutomationEventHandler, public IUIAutomationFocusChangedEventHandler, public IUIAutomationPropertyChangedEventHandler, public IUIAutomationNotificationEventHandler, public IUIAutomationActiveTextPositionChangedEventHandler { private: unsigned long m_refCount; @@ -32,6 +38,7 @@ class RateLimitedEventHandler: public IUIAutomationEventHandler, public IUIAutom std::mutex mtx; std::list m_eventRecords; std::map, std::pair> m_eventRecordsByKey; + FlushRequest lastFlushRequest = FlushRequest::none; template HRESULT queueEvent(EventRecordArgTypes&&... args); From c1a436ef5caa26ba333095bdd1dbb57abab04ea1 Mon Sep 17 00:00:00 2001 From: Michael Curran Date: Tue, 2 May 2023 19:30:06 +1000 Subject: [PATCH 09/10] UIAHandler: execute the UIA event limiter flush on NVDA's UIAHandler thread rather than NVDA's main thread. This works around some weird freezing when accessing NVDA's own UI. Note that UIA core always calls UIA event handlers on worker threads, except for when processing events within a UI Automation event registration call. Thus, our UIA Handler MTA thread sits there doing nothing 99% of the time. So moving the flush calls into this thread does not disrrupt true UI Automation event handler threads. If this design seems okay, then we may consider registering a custom window on the UIAHandler thread in order to receive posted messages directly, rather than them being queued to UIAHandler's queue from NVDA's main thread. --- source/UIAHandler/__init__.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/source/UIAHandler/__init__.py b/source/UIAHandler/__init__.py index 0057ec30492..d5388dee1d8 100644 --- a/source/UIAHandler/__init__.py +++ b/source/UIAHandler/__init__.py @@ -1411,8 +1411,10 @@ def _isDebug(): def eventLimiterWindowProc(msg, wParam, lParam): + if not handler: return if msg == WM_NVDA_UIA_FLUSH: + func = lambda: NVDAHelper.localLib.rateLimitedUIAEventHandler_flush(wParam) if lParam == 0: - NVDAHelper.localLib.rateLimitedUIAEventHandler_flush(wParam) + handler.MTAThreadQueue.put(func) else: - core.callLater(lParam, NVDAHelper.localLib.rateLimitedUIAEventHandler_flush, wParam) + core.callLater(lParam, handler.MTAThreadQueue.put, func) From b94f4a11aec3fd0ba468da80b3fe56533a5eabab Mon Sep 17 00:00:00 2001 From: Michael Curran Date: Mon, 8 May 2023 17:34:15 +1000 Subject: [PATCH 10/10] UIA event limiter: * rather than the limiter sending a window message to NvDA to request a flush, instead the limiter class has its own dedicated thread which it wakes for flushing. * Now all UI Automation events are coalesced. However, there is no delay before flushing. It will happen as soon as the flush thread can wake and flush. These changes mean that UI Automation events again enter NVDA on a dedicated MTA thread, with no deliberate delay or having to go through NvDA's main thread first. Therefore the primary reason for the event limiter in c++ is to ensure that UI Automation core is never blocked on an event handler - they should queue and return very fast and not hit Python at all. Secondarily, any duplicate UI Automation events that exist before NVDA receives them are removed. This all still has the major advantage that doing `git -P log -1000` in a Windows console will not take out UIA focus events, nor slow down NvDA while the console isn't in focus. There could be further work in UIAHandler to stop adding duplicate textchange and caret UIA events to NVDA's event queue if the last one has not yet been processed. --- nvdaHelper/local/UIAEventLimiter/api.cpp | 16 +-- .../local/UIAEventLimiter/eventRecord.h | 50 +++++-- .../rateLimitedEventHandler.cpp | 133 +++++++++--------- .../UIAEventLimiter/rateLimitedEventHandler.h | 40 ++++-- nvdaHelper/local/UIAEventLimiter/utils.h | 10 ++ source/UIAHandler/__init__.py | 19 +-- 6 files changed, 145 insertions(+), 123 deletions(-) diff --git a/nvdaHelper/local/UIAEventLimiter/api.cpp b/nvdaHelper/local/UIAEventLimiter/api.cpp index 73ec92439be..ba857690bc4 100644 --- a/nvdaHelper/local/UIAEventLimiter/api.cpp +++ b/nvdaHelper/local/UIAEventLimiter/api.cpp @@ -16,15 +16,15 @@ This license can be found at: #include #include "rateLimitedEventHandler.h" -HRESULT rateLimitedUIAEventHandler_create(IUnknown* pExistingHandler, HWND messageWindow, UINT flushMessage, RateLimitedEventHandler** ppRateLimitedEventHandler) { +HRESULT rateLimitedUIAEventHandler_create(IUnknown* pExistingHandler, RateLimitedEventHandler** ppRateLimitedEventHandler) { LOG_DEBUG(L"rateLimitedUIAEventHandler_create called"); - if(!pExistingHandler || !messageWindow || !ppRateLimitedEventHandler) { + if(!pExistingHandler || !ppRateLimitedEventHandler) { LOG_ERROR(L"rateLimitedUIAEventHandler_create: one or more NULL arguments"); return E_INVALIDARG; } // Create the RateLimitedEventHandler instance - *ppRateLimitedEventHandler = new RateLimitedEventHandler(pExistingHandler, messageWindow, flushMessage); + *ppRateLimitedEventHandler = new RateLimitedEventHandler(pExistingHandler); if (!(*ppRateLimitedEventHandler)) { LOG_ERROR(L"rateLimitedUIAEventHandler_create: Could not create RateLimitedUIAEventHandler. Returning"); return E_OUTOFMEMORY; @@ -32,13 +32,3 @@ HRESULT rateLimitedUIAEventHandler_create(IUnknown* pExistingHandler, HWND messa LOG_DEBUG(L"rateLimitedUIAEventHandler_create: done"); return S_OK; } - -HRESULT rateLimitedUIAEventHandler_flush(RateLimitedEventHandler* pRateLimitedEventHandler) { - LOG_DEBUG(L"rateLimitedUIAEventHandler_flush called"); - if(!pRateLimitedEventHandler) { - LOG_ERROR(L"rateLimitedUIAEventHandler_flush: invalid argument. Returning"); - return E_INVALIDARG; - } - pRateLimitedEventHandler->flush(); - return S_OK; -} diff --git a/nvdaHelper/local/UIAEventLimiter/eventRecord.h b/nvdaHelper/local/UIAEventLimiter/eventRecord.h index 42aa7f18c9a..23b85a9f4ff 100644 --- a/nvdaHelper/local/UIAEventLimiter/eventRecord.h +++ b/nvdaHelper/local/UIAEventLimiter/eventRecord.h @@ -20,8 +20,15 @@ This license can be found at: #include #include "utils.h" +// The following structs are used to represent UI Automation event params. +// A part from holding the params, +// each struct also contains a method for generating a comparison key +// which is used to detect and remove duplicate events. +// The key is made up of the element's runtime ID, +// plus any extra event params that make the vent unique, +// E.g. event ID, property ID etc. + struct AutomationEventRecord_t { - static const bool isCoalesceable = true; CComPtr sender; EVENTID eventID; std::vector generateCoalescingKey() const { @@ -32,7 +39,6 @@ struct AutomationEventRecord_t { }; struct PropertyChangedEventRecord_t { - static const bool isCoalesceable = false; CComPtr sender; PROPERTYID propertyID; CComVariant newValue; @@ -45,35 +51,57 @@ struct PropertyChangedEventRecord_t { }; struct FocusChangedEventRecord_t { - static const bool isCoalesceable = false; CComPtr sender; + std::vector generateCoalescingKey() const { + auto key = getRuntimeIDFromElement(sender); + key.push_back(UIA_AutomationFocusChangedEventId); + return key; + } }; struct NotificationEventRecord_t { - static const bool isCoalesceable = false; CComPtr sender; NotificationKind notificationKind; NotificationProcessing notificationProcessing; CComBSTR displayString; CComBSTR activityID; + std::vector generateCoalescingKey() const { + auto key = getRuntimeIDFromElement(sender); + key.push_back(UIA_NotificationEventId); + key.push_back(notificationKind); + key.push_back(notificationProcessing); + // Include the activity ID in the key also, + // by converting it to a sequence of ints. + if(activityID.m_str) { + for(int c: std::wstring_view(activityID.m_str)) { + key.push_back(c); + } + } else { + key.push_back(0); + } + return key; + } }; struct ActiveTextPositionChangedEventRecord_t { - static const bool isCoalesceable = false; CComPtr sender; CComPtr range; + std::vector generateCoalescingKey() const { + auto key = getRuntimeIDFromElement(sender); + key.push_back(UIA_ActiveTextPositionChangedEventId); + return key; + } }; +// @brief a variant type that holds all possible UI Automation event records we support. using EventRecordVariant_t = std::variant; +// @brief A concept to be used with the above event record types +// that ensures the type has a generateCoalescingKey method, +// and that the type appears in the EventRecordVariant_t type. template concept EventRecordConstraints = requires(T t) { - // type must have a static const bool member called 'isCoalesceable' - { t.isCoalesceable } -> std::same_as; - // if 'isCoaleseable' is true, then the type must have a 'generateCoalescingKey' method. - requires (T::isCoalesceable == false) || requires(T t) { - { t.generateCoalescingKey() } -> std::same_as>; - }; + { t.generateCoalescingKey() } -> std::same_as>; // The type must be supported by the EventRecordVariant_t variant type requires supports_alternative; }; diff --git a/nvdaHelper/local/UIAEventLimiter/rateLimitedEventHandler.cpp b/nvdaHelper/local/UIAEventLimiter/rateLimitedEventHandler.cpp index dbc68590af4..b2a31138ad5 100644 --- a/nvdaHelper/local/UIAEventLimiter/rateLimitedEventHandler.cpp +++ b/nvdaHelper/local/UIAEventLimiter/rateLimitedEventHandler.cpp @@ -30,60 +30,62 @@ This license can be found at: template HRESULT RateLimitedEventHandler::queueEvent(EventRecordArgTypes&&... args) { LOG_DEBUG(L"RateLimitedUIAEventHandler::queueEvent called"); - FlushRequest flushRequest = FlushRequest::none; - unsigned int flushTimeMS = 0; - { std::lock_guard lock(mtx); + bool needsFlush = false; + { std::lock_guard lock(m_mtx); // work out whether we need to request a flush after inserting this event. - if(!EventRecordClass::isCoalesceable) { - // not a coalesceable event so requires a quick flush - // if a quick flush has not been requested already. - if(lastFlushRequest < FlushRequest::quick) { - LOG_DEBUG(L"RateLimitedUIAEventHandler::queueEvent: requesting quick flush after queuing"); - lastFlushRequest = flushRequest = FlushRequest::quick; - } else { - LOG_DEBUG(L"RateLimitedUIAEventHandler::queueEvent: quick flush needed, but quick flush already requested."); - } - } else if(m_eventRecords.empty()) { - // It is a coalesceable event and its the first event since the last flush, - // so requires a delayed flush - // if a delayed or quick flush is not requested already. - if(lastFlushRequest < FlushRequest::delayed) { - LOG_DEBUG(L"RateLimitedUIAEventHandler::queueEvent: requesting delayed flush after queuing"); - lastFlushRequest = flushRequest = FlushRequest::delayed; - flushTimeMS = 30; - } else { - LOG_DEBUG(L"RateLimitedUIAEventHandler::queueEvent: delayed flush needed, but quick flush already requested."); - } - } + needsFlush = m_eventRecords.empty(); LOG_DEBUG(L"RateLimitedUIAEventHandler::queueEvent: Inserting new event"); auto& recordVar = m_eventRecords.emplace_back(std::in_place_type_t{}, args...); auto recordVarIter = m_eventRecords.end(); recordVarIter--; auto& record = std::get(recordVar); - if constexpr(EventRecordClass::isCoalesceable) { - LOG_DEBUG(L"RateLimitedUIAEventHandler::queueEvent: Is a coalesceable event"); - auto coalescingKey = record.generateCoalescingKey(); - auto existingKeyIter = m_eventRecordsByKey.find(coalescingKey); - if(existingKeyIter != m_eventRecordsByKey.end()) { - LOG_DEBUG(L"RateLimitedUIAEventHandler::queueEvent: found existing event with same key"); - auto& [existingRecordVarIter,existingCoalesceCount] = existingKeyIter->second; - LOG_DEBUG(L"RateLimitedUIAEventHandler::queueEvent: updating key and count to "<<(existingCoalesceCount+1)); - existingKeyIter->second = {recordVarIter, existingCoalesceCount + 1}; - LOG_DEBUG(L"RateLimitedUIAEventHandler::queueEvent: erasing old item"); - m_eventRecords.erase(existingRecordVarIter); - } else { - LOG_DEBUG(L"RateLimitedUIAEventHandler::queueEvent: Adding key"); - m_eventRecordsByKey.insert_or_assign(coalescingKey, std::pair(recordVarIter, 1)); - } + auto coalescingKey = record.generateCoalescingKey(); + auto existingKeyIter = m_eventRecordsByKey.find(coalescingKey); + if(existingKeyIter != m_eventRecordsByKey.end()) { + LOG_DEBUG(L"RateLimitedUIAEventHandler::queueEvent: found existing event with same key"); + auto& [existingRecordVarIter,existingCoalesceCount] = existingKeyIter->second; + LOG_DEBUG(L"RateLimitedUIAEventHandler::queueEvent: updating key and count to "<<(existingCoalesceCount+1)); + existingKeyIter->second = {recordVarIter, existingCoalesceCount + 1}; + LOG_DEBUG(L"RateLimitedUIAEventHandler::queueEvent: erasing old item"); + m_eventRecords.erase(existingRecordVarIter); + } else { + LOG_DEBUG(L"RateLimitedUIAEventHandler::queueEvent: Adding key"); + m_eventRecordsByKey.insert_or_assign(coalescingKey, std::pair(recordVarIter, 1)); } - } - if(flushRequest > FlushRequest::none) { - LOG_DEBUG(L"RateLimitedUIAEventHandler::queueEvent: posting flush message with delay of "<(this), flushTimeMS); - } + if(needsFlush) { + LOG_DEBUG(L"RateLimitedUIAEventHandler::queueEvent: requesting flush"); + m_needsFlush = true; + m_flushConditionVar.notify_one(); + } + } // m_mtx released. return S_OK; } +void RateLimitedEventHandler::flusherThreadFunc(std::stop_token stopToken) { + LOG_DEBUG(L"flusherThread started"); + // If this thread is requested to stop, we need to ensure we wake up. + std::stop_callback stopCallback{stopToken, [this](){ + this->m_flushConditionVar.notify_all(); + }}; + do { // thread main loop + LOG_DEBUG(L"flusherThreadFunc sleeping..."); + { std::unique_lock lock(m_mtx); + // sleep until a flush is needed or this thread should stop. + m_flushConditionVar.wait(lock, [this, stopToken](){ + return this->m_needsFlush || stopToken.stop_requested(); + }); + LOG_DEBUG(L"flusherThread woke up"); + if(stopToken.stop_requested()) { + LOG_DEBUG(L"flusherThread returning as stop requested"); + return; + } + m_needsFlush = false; + } // m_mtx released here. + flushEvents(); + } while(!stopToken.stop_requested()); + LOG_DEBUG(L"flusherThread returning"); +} + HRESULT RateLimitedEventHandler::emitEvent(const AutomationEventRecord_t& record) const { LOG_DEBUG(L"RateLimitedUIAEventHandler::emitAutomationEvent called"); if(!m_pExistingAutomationEventHandler) { @@ -100,6 +102,7 @@ HRESULT RateLimitedEventHandler::emitEvent(const FocusChangedEventRecord_t& reco LOG_ERROR(L"RateLimitedUIAEventHandler::emitFocusChangedEvent: interface not supported."); return E_NOINTERFACE; } + LOG_DEBUG(L"Emitting focus changed event"); return m_pExistingFocusChangedEventHandler->HandleFocusChangedEvent(record.sender); } @@ -109,6 +112,7 @@ HRESULT RateLimitedEventHandler::emitEvent(const PropertyChangedEventRecord_t& r LOG_ERROR(L"RateLimitedUIAEventHandler::emitPropertyChangedEvent: interface not supported."); return E_NOINTERFACE; } + LOG_DEBUG(L"Emitting property changed event for property "<<(record.propertyID)); return m_pExistingPropertyChangedEventHandler->HandlePropertyChangedEvent(record.sender, record.propertyID, record.newValue); } @@ -118,6 +122,7 @@ HRESULT RateLimitedEventHandler::emitEvent(const NotificationEventRecord_t& reco LOG_ERROR(L"RateLimitedUIAEventHandler::emitNotificationChangedEvent: interface not supported."); return E_NOINTERFACE; } + LOG_DEBUG(L"Emitting notification event"); return m_pExistingNotificationEventHandler->HandleNotificationEvent(record.sender, record.notificationKind, record.notificationProcessing, record.displayString, record.activityID); } @@ -127,6 +132,7 @@ HRESULT RateLimitedEventHandler::emitEvent(const ActiveTextPositionChangedEventR LOG_ERROR(L"RateLimitedUIAEventHandler::emitActiveTextPositionChangedEvent: interface not supported."); return E_NOINTERFACE; } + LOG_DEBUG(L"Emitting active text position changed event"); return m_pExistingActiveTextPositionChangedEventHandler->HandleActiveTextPositionChangedEvent(record.sender, record.range); } @@ -134,21 +140,26 @@ RateLimitedEventHandler::~RateLimitedEventHandler() { LOG_DEBUG(L"RateLimitedUIAEventHandler::~RateLimitedUIAEventHandler called"); } -RateLimitedEventHandler::RateLimitedEventHandler(IUnknown* pExistingHandler, HWND messageWindow, UINT flushMessage) - : m_messageWindow(messageWindow), m_flushMessage(flushMessage), m_refCount(1), m_pExistingAutomationEventHandler(pExistingHandler), m_pExistingFocusChangedEventHandler(pExistingHandler), m_pExistingPropertyChangedEventHandler(pExistingHandler), m_pExistingNotificationEventHandler(pExistingHandler), m_pExistingActiveTextPositionChangedEventHandler(pExistingHandler) { +RateLimitedEventHandler::RateLimitedEventHandler(IUnknown* pExistingHandler): + m_pExistingAutomationEventHandler(pExistingHandler), m_pExistingFocusChangedEventHandler(pExistingHandler), m_pExistingPropertyChangedEventHandler(pExistingHandler), m_pExistingNotificationEventHandler(pExistingHandler), m_pExistingActiveTextPositionChangedEventHandler(pExistingHandler), + m_flusherThread([this](std::stop_token st){ this->flusherThreadFunc(st); }) +{ LOG_DEBUG(L"RateLimitedUIAEventHandler::RateLimitedUIAEventHandler called"); } // IUnknown methods ULONG STDMETHODCALLTYPE RateLimitedEventHandler::AddRef() { - return InterlockedIncrement(&m_refCount); + auto refCount = InterlockedIncrement(&m_refCount); + LOG_DEBUG(L"AddRef: "<(sender, range); } -void RateLimitedEventHandler::flush() { - LOG_DEBUG(L"RateLimitedUIAEventHandler::flush called"); +void RateLimitedEventHandler::flushEvents() { + LOG_DEBUG(L"RateLimitedEventHandler::flushEvents called"); decltype(m_eventRecords) eventRecordsCopy; decltype(m_eventRecordsByKey) eventRecordsByKeyCopy; - { std::lock_guard lock(mtx); + { std::lock_guard lock{m_mtx}; eventRecordsCopy.swap(m_eventRecords); eventRecordsByKeyCopy.swap(m_eventRecordsByKey); - lastFlushRequest = FlushRequest::none; - } - + } // m_mtx released here. // Emit events - LOG_DEBUG(L"RateLimitedUIAEventHandler::flush: Emitting events..."); + LOG_DEBUG(L"RateLimitedUIAEventHandler::flusherThreadFunc: Emitting events..."); for(const auto& recordVar: eventRecordsCopy) { std::visit([this](const auto& record) { this->emitEvent(record); }, recordVar); } - /* - unsigned int count = std::accumulate(eventRecordsByKeyCopy.begin(), eventRecordsByKeyCopy.end(), 0, [](const auto& acc, const auto& i) { - auto count = i.second.second; - if(count > 1) { - return acc + count; - } - return acc; - }); - if(count > 0) { - Beep(440 + (count*10), 40); - } - */ - LOG_DEBUG(L"RateLimitedUIAEventHandler::flush: done emitting events"); + LOG_DEBUG(L"RateLimitedUIAEventHandler::flusherThreadFunc: Done emitting events"); } diff --git a/nvdaHelper/local/UIAEventLimiter/rateLimitedEventHandler.h b/nvdaHelper/local/UIAEventLimiter/rateLimitedEventHandler.h index 4fd9baba6a3..1930829a4f8 100644 --- a/nvdaHelper/local/UIAEventLimiter/rateLimitedEventHandler.h +++ b/nvdaHelper/local/UIAEventLimiter/rateLimitedEventHandler.h @@ -19,40 +19,56 @@ This license can be found at: #include #include "eventRecord.h" -enum FlushRequest { - none, - delayed, - quick -}; +// @brief a class that listens for various UI Automation events, +// stores them in in an internal queue (removing any duplicates), +// and sends them onto an existing UI Automation event handler in a separate thread. +// This ensures that UI Automation core is never blocked while sending events to this class. class RateLimitedEventHandler: public IUIAutomationEventHandler, public IUIAutomationFocusChangedEventHandler, public IUIAutomationPropertyChangedEventHandler, public IUIAutomationNotificationEventHandler, public IUIAutomationActiveTextPositionChangedEventHandler { private: - unsigned long m_refCount; + unsigned long m_refCount = 1; CComQIPtr m_pExistingAutomationEventHandler; CComQIPtr m_pExistingFocusChangedEventHandler; CComQIPtr m_pExistingPropertyChangedEventHandler; CComQIPtr m_pExistingNotificationEventHandler; CComQIPtr m_pExistingActiveTextPositionChangedEventHandler; - HWND m_messageWindow; - UINT m_flushMessage; - std::mutex mtx; std::list m_eventRecords; std::map, std::pair> m_eventRecordsByKey; - FlushRequest lastFlushRequest = FlushRequest::none; + bool m_needsFlush = false; + std::mutex m_mtx; + std::condition_variable m_flushConditionVar; + std::jthread m_flusherThread; + + /// @brief a thread function that wakes and flushes the event queue when one or more events have been added. + /// @param stopToken used to check if the thread should stop. + void flusherThreadFunc(std::stop_token stopToken); + /// @brief a template function that queues a UI Automation event. + /// @tparam EventRecordClass the type of event record representing a UI Automation event. + /// @tparam ...EventRecordArgTypes the argument types required to construct the event record + /// @param ...args the arguments to construct the event record. + /// @return S_OK on success or a failure code otherwise. template HRESULT queueEvent(EventRecordArgTypes&&... args); + /// @brief Emits a UI Automation event to its existing handler. + /// @param record the event record representing the UI automation event. + /// @return S_OK on success or a failure code otherwise. HRESULT emitEvent(const AutomationEventRecord_t& record) const; HRESULT emitEvent(const FocusChangedEventRecord_t& record) const; HRESULT emitEvent(const PropertyChangedEventRecord_t& record) const; HRESULT emitEvent(const NotificationEventRecord_t& record) const; HRESULT emitEvent(const ActiveTextPositionChangedEventRecord_t& record) const; + /// @brief Removes all events from the queue, sending them on to the existing UI automation event handler. + void flushEvents(); + ~RateLimitedEventHandler(); public: - RateLimitedEventHandler(IUnknown* pExistingHandler, HWND messageWindow, UINT flushMessage); + /// @brief class constructor. + /// @param pExistingHandler a pointer to an existing UI Automation event handler where events should be sent after they are flushed from the queue. + RateLimitedEventHandler(IUnknown* pExistingHandler); // IUnknown methods ULONG STDMETHODCALLTYPE AddRef(); @@ -66,6 +82,4 @@ class RateLimitedEventHandler: public IUIAutomationEventHandler, public IUIAutom HRESULT STDMETHODCALLTYPE HandleNotificationEvent(IUIAutomationElement* sender, NotificationKind notificationKind, NotificationProcessing notificationProcessing, BSTR displayString, BSTR activityID); HRESULT STDMETHODCALLTYPE HandleActiveTextPositionChangedEvent(IUIAutomationElement* sender, IUIAutomationTextRange* range); - void flush(); - }; diff --git a/nvdaHelper/local/UIAEventLimiter/utils.h b/nvdaHelper/local/UIAEventLimiter/utils.h index 2a11ef89ef3..2e55486e627 100644 --- a/nvdaHelper/local/UIAEventLimiter/utils.h +++ b/nvdaHelper/local/UIAEventLimiter/utils.h @@ -20,14 +20,24 @@ This license can be found at: #include #include +/// @brief creates a vector of ints from a SAFEARRAY. +/// @param pSafeArray +/// @return the vector of ints. std::vector SafeArrayToVector(SAFEARRAY* pSafeArray); +/// @brief Fetches the runtimeID from a given uI Automation element. +/// @param pElement the UI Automation element whose runtime ID should be fetched. +/// @return the runtime ID from the element. std::vector getRuntimeIDFromElement(IUIAutomationElement* pElement); +// @brief a helper template function for the supports_alternative concept. template constexpr bool supports_alternative_impl(std::index_sequence) { return (std::same_as> || ...); } +// @brief a concept that checks if a given type can be held by a given variant type. +// @tparam T the type to check +// @tparam V the variant type to check template concept supports_alternative = supports_alternative_impl(std::make_index_sequence>{}); diff --git a/source/UIAHandler/__init__.py b/source/UIAHandler/__init__.py index d5388dee1d8..c8c811f152f 100644 --- a/source/UIAHandler/__init__.py +++ b/source/UIAHandler/__init__.py @@ -27,7 +27,6 @@ import threading import time -import winAPI.messageWindow import IAccessibleHandler.internalWinEventHandler import config from config import ( @@ -449,7 +448,7 @@ def terminate(self): ) ) self.MTAThreadQueue.put_nowait(None) - #Wait for the MTA thread to die (while still message pumping) + # Wait for the MTA thread to die (while still message pumping) if windll.user32.MsgWaitForMultipleObjects(1,byref(MTAThreadHandle),False,200,0)!=0: log.debugWarning("Timeout or error while waiting for UIAHandler MTA thread") windll.kernel32.CloseHandle(MTAThreadHandle) @@ -517,11 +516,8 @@ def MTAThreadFunc(self): self.pRateLimitedEventHandler = POINTER(IUnknown)() NVDAHelper.localLib.rateLimitedUIAEventHandler_create( self._com_pointers_[IUnknown._iid_], - core.messageWindow.handle, - WM_NVDA_UIA_FLUSH, byref(self.pRateLimitedEventHandler) ) - winAPI.messageWindow.pre_handleWindowMessage.register(eventLimiterWindowProc) if utils._shouldSelectivelyRegister(): self._createLocalEventHandlerGroup() self._registerGlobalEventHandlers() @@ -1405,16 +1401,3 @@ def terminate(): def _isDebug(): return config.conf["debugLog"]["UIA"] - - -WM_NVDA_UIA_FLUSH = winUser.registerWindowMessage("WM_NVDA_UIA_FLUSH") - - -def eventLimiterWindowProc(msg, wParam, lParam): - if not handler: return - if msg == WM_NVDA_UIA_FLUSH: - func = lambda: NVDAHelper.localLib.rateLimitedUIAEventHandler_flush(wParam) - if lParam == 0: - handler.MTAThreadQueue.put(func) - else: - core.callLater(lParam, handler.MTAThreadQueue.put, func)