diff --git a/nvdaHelper/local/UIAEventLimiter/api.cpp b/nvdaHelper/local/UIAEventLimiter/api.cpp new file mode 100644 index 00000000000..ba857690bc4 --- /dev/null +++ b/nvdaHelper/local/UIAEventLimiter/api.cpp @@ -0,0 +1,34 @@ +/* +This file is a part of the NVDA project. +URL: http://github.com/nvaccess/nvda/ +Copyright 2023 NV Access Limited. + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License version 2.0, as published by + the Free Software Foundation. + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +This license can be found at: +http://www.gnu.org/licenses/old-licenses/gpl-2.0.html +*/ + +#include +#include +#include "rateLimitedEventHandler.h" + +HRESULT rateLimitedUIAEventHandler_create(IUnknown* pExistingHandler, RateLimitedEventHandler** ppRateLimitedEventHandler) { + LOG_DEBUG(L"rateLimitedUIAEventHandler_create called"); + if(!pExistingHandler || !ppRateLimitedEventHandler) { + LOG_ERROR(L"rateLimitedUIAEventHandler_create: one or more NULL arguments"); + return E_INVALIDARG; + } + + // Create the RateLimitedEventHandler instance + *ppRateLimitedEventHandler = new RateLimitedEventHandler(pExistingHandler); + if (!(*ppRateLimitedEventHandler)) { + LOG_ERROR(L"rateLimitedUIAEventHandler_create: Could not create RateLimitedUIAEventHandler. Returning"); + return E_OUTOFMEMORY; + } + LOG_DEBUG(L"rateLimitedUIAEventHandler_create: done"); + return S_OK; +} diff --git a/nvdaHelper/local/UIAEventLimiter/eventRecord.h b/nvdaHelper/local/UIAEventLimiter/eventRecord.h new file mode 100644 index 00000000000..23b85a9f4ff --- /dev/null +++ b/nvdaHelper/local/UIAEventLimiter/eventRecord.h @@ -0,0 +1,107 @@ +/* +This file is a part of the NVDA project. +URL: http://github.com/nvaccess/nvda/ +Copyright 2023 NV Access Limited. + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License version 2.0, as published by + the Free Software Foundation. + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +This license can be found at: +http://www.gnu.org/licenses/old-licenses/gpl-2.0.html +*/ + +#pragma once + +#include +#include +#include +#include +#include "utils.h" + +// The following structs are used to represent UI Automation event params. +// A part from holding the params, +// each struct also contains a method for generating a comparison key +// which is used to detect and remove duplicate events. +// The key is made up of the element's runtime ID, +// plus any extra event params that make the vent unique, +// E.g. event ID, property ID etc. + +struct AutomationEventRecord_t { + CComPtr sender; + EVENTID eventID; + std::vector generateCoalescingKey() const { + auto key = getRuntimeIDFromElement(sender); + key.push_back(eventID); + return key; + } +}; + +struct PropertyChangedEventRecord_t { + CComPtr sender; + PROPERTYID propertyID; + CComVariant newValue; + std::vector generateCoalescingKey() const { + auto key = getRuntimeIDFromElement(sender); + key.push_back(UIA_AutomationPropertyChangedEventId); + key.push_back(propertyID); + return key; + } +}; + +struct FocusChangedEventRecord_t { + CComPtr sender; + std::vector generateCoalescingKey() const { + auto key = getRuntimeIDFromElement(sender); + key.push_back(UIA_AutomationFocusChangedEventId); + return key; + } +}; + +struct NotificationEventRecord_t { + CComPtr sender; + NotificationKind notificationKind; + NotificationProcessing notificationProcessing; + CComBSTR displayString; + CComBSTR activityID; + std::vector generateCoalescingKey() const { + auto key = getRuntimeIDFromElement(sender); + key.push_back(UIA_NotificationEventId); + key.push_back(notificationKind); + key.push_back(notificationProcessing); + // Include the activity ID in the key also, + // by converting it to a sequence of ints. + if(activityID.m_str) { + for(int c: std::wstring_view(activityID.m_str)) { + key.push_back(c); + } + } else { + key.push_back(0); + } + return key; + } +}; + +struct ActiveTextPositionChangedEventRecord_t { + CComPtr sender; + CComPtr range; + std::vector generateCoalescingKey() const { + auto key = getRuntimeIDFromElement(sender); + key.push_back(UIA_ActiveTextPositionChangedEventId); + return key; + } +}; + +// @brief a variant type that holds all possible UI Automation event records we support. +using EventRecordVariant_t = std::variant; + +// @brief A concept to be used with the above event record types +// that ensures the type has a generateCoalescingKey method, +// and that the type appears in the EventRecordVariant_t type. +template +concept EventRecordConstraints = requires(T t) { + { t.generateCoalescingKey() } -> std::same_as>; + // The type must be supported by the EventRecordVariant_t variant type + requires supports_alternative; +}; diff --git a/nvdaHelper/local/UIAEventLimiter/rateLimitedEventHandler.cpp b/nvdaHelper/local/UIAEventLimiter/rateLimitedEventHandler.cpp new file mode 100644 index 00000000000..b2a31138ad5 --- /dev/null +++ b/nvdaHelper/local/UIAEventLimiter/rateLimitedEventHandler.cpp @@ -0,0 +1,243 @@ +/* +This file is a part of the NVDA project. +URL: http://github.com/nvaccess/nvda/ +Copyright 2023 NV Access Limited. + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License version 2.0, as published by + the Free Software Foundation. + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +This license can be found at: +http://www.gnu.org/licenses/old-licenses/gpl-2.0.html +*/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "eventRecord.h" +#include "rateLimitedEventHandler.h" + +template +HRESULT RateLimitedEventHandler::queueEvent(EventRecordArgTypes&&... args) { + LOG_DEBUG(L"RateLimitedUIAEventHandler::queueEvent called"); + bool needsFlush = false; + { std::lock_guard lock(m_mtx); + // work out whether we need to request a flush after inserting this event. + needsFlush = m_eventRecords.empty(); + LOG_DEBUG(L"RateLimitedUIAEventHandler::queueEvent: Inserting new event"); + auto& recordVar = m_eventRecords.emplace_back(std::in_place_type_t{}, args...); + auto recordVarIter = m_eventRecords.end(); + recordVarIter--; + auto& record = std::get(recordVar); + auto coalescingKey = record.generateCoalescingKey(); + auto existingKeyIter = m_eventRecordsByKey.find(coalescingKey); + if(existingKeyIter != m_eventRecordsByKey.end()) { + LOG_DEBUG(L"RateLimitedUIAEventHandler::queueEvent: found existing event with same key"); + auto& [existingRecordVarIter,existingCoalesceCount] = existingKeyIter->second; + LOG_DEBUG(L"RateLimitedUIAEventHandler::queueEvent: updating key and count to "<<(existingCoalesceCount+1)); + existingKeyIter->second = {recordVarIter, existingCoalesceCount + 1}; + LOG_DEBUG(L"RateLimitedUIAEventHandler::queueEvent: erasing old item"); + m_eventRecords.erase(existingRecordVarIter); + } else { + LOG_DEBUG(L"RateLimitedUIAEventHandler::queueEvent: Adding key"); + m_eventRecordsByKey.insert_or_assign(coalescingKey, std::pair(recordVarIter, 1)); + } + if(needsFlush) { + LOG_DEBUG(L"RateLimitedUIAEventHandler::queueEvent: requesting flush"); + m_needsFlush = true; + m_flushConditionVar.notify_one(); + } + } // m_mtx released. + return S_OK; +} + +void RateLimitedEventHandler::flusherThreadFunc(std::stop_token stopToken) { + LOG_DEBUG(L"flusherThread started"); + // If this thread is requested to stop, we need to ensure we wake up. + std::stop_callback stopCallback{stopToken, [this](){ + this->m_flushConditionVar.notify_all(); + }}; + do { // thread main loop + LOG_DEBUG(L"flusherThreadFunc sleeping..."); + { std::unique_lock lock(m_mtx); + // sleep until a flush is needed or this thread should stop. + m_flushConditionVar.wait(lock, [this, stopToken](){ + return this->m_needsFlush || stopToken.stop_requested(); + }); + LOG_DEBUG(L"flusherThread woke up"); + if(stopToken.stop_requested()) { + LOG_DEBUG(L"flusherThread returning as stop requested"); + return; + } + m_needsFlush = false; + } // m_mtx released here. + flushEvents(); + } while(!stopToken.stop_requested()); + LOG_DEBUG(L"flusherThread returning"); +} + +HRESULT RateLimitedEventHandler::emitEvent(const AutomationEventRecord_t& record) const { + LOG_DEBUG(L"RateLimitedUIAEventHandler::emitAutomationEvent called"); + if(!m_pExistingAutomationEventHandler) { + LOG_ERROR(L"RateLimitedUIAEventHandler::emitAutomationEvent: interface not supported."); + return E_NOINTERFACE; + } + LOG_DEBUG(L"Emitting automationEvent for eventID "<HandleAutomationEvent(record.sender, record.eventID); +} + +HRESULT RateLimitedEventHandler::emitEvent(const FocusChangedEventRecord_t& record) const { + LOG_DEBUG(L"RateLimitedUIAEventHandler::emitFocusChangedEvent called"); + if(!m_pExistingFocusChangedEventHandler) { + LOG_ERROR(L"RateLimitedUIAEventHandler::emitFocusChangedEvent: interface not supported."); + return E_NOINTERFACE; + } + LOG_DEBUG(L"Emitting focus changed event"); + return m_pExistingFocusChangedEventHandler->HandleFocusChangedEvent(record.sender); +} + +HRESULT RateLimitedEventHandler::emitEvent(const PropertyChangedEventRecord_t& record) const { + LOG_DEBUG(L"RateLimitedUIAEventHandler::emitPropertyChangedEvent called"); + if(!m_pExistingPropertyChangedEventHandler) { + LOG_ERROR(L"RateLimitedUIAEventHandler::emitPropertyChangedEvent: interface not supported."); + return E_NOINTERFACE; + } + LOG_DEBUG(L"Emitting property changed event for property "<<(record.propertyID)); + return m_pExistingPropertyChangedEventHandler->HandlePropertyChangedEvent(record.sender, record.propertyID, record.newValue); +} + +HRESULT RateLimitedEventHandler::emitEvent(const NotificationEventRecord_t& record) const { + LOG_DEBUG(L"RateLimitedUIAEventHandler::emitNotificationEvent called"); + if(!m_pExistingNotificationEventHandler) { + LOG_ERROR(L"RateLimitedUIAEventHandler::emitNotificationChangedEvent: interface not supported."); + return E_NOINTERFACE; + } + LOG_DEBUG(L"Emitting notification event"); + return m_pExistingNotificationEventHandler->HandleNotificationEvent(record.sender, record.notificationKind, record.notificationProcessing, record.displayString, record.activityID); +} + +HRESULT RateLimitedEventHandler::emitEvent(const ActiveTextPositionChangedEventRecord_t& record) const { + LOG_DEBUG(L"RateLimitedUIAEventHandler::emitActiveTextPositionChangedEvent called"); + if(!m_pExistingActiveTextPositionChangedEventHandler) { + LOG_ERROR(L"RateLimitedUIAEventHandler::emitActiveTextPositionChangedEvent: interface not supported."); + return E_NOINTERFACE; + } + LOG_DEBUG(L"Emitting active text position changed event"); + return m_pExistingActiveTextPositionChangedEventHandler->HandleActiveTextPositionChangedEvent(record.sender, record.range); +} + +RateLimitedEventHandler::~RateLimitedEventHandler() { + LOG_DEBUG(L"RateLimitedUIAEventHandler::~RateLimitedUIAEventHandler called"); +} + +RateLimitedEventHandler::RateLimitedEventHandler(IUnknown* pExistingHandler): + m_pExistingAutomationEventHandler(pExistingHandler), m_pExistingFocusChangedEventHandler(pExistingHandler), m_pExistingPropertyChangedEventHandler(pExistingHandler), m_pExistingNotificationEventHandler(pExistingHandler), m_pExistingActiveTextPositionChangedEventHandler(pExistingHandler), + m_flusherThread([this](std::stop_token st){ this->flusherThreadFunc(st); }) +{ + LOG_DEBUG(L"RateLimitedUIAEventHandler::RateLimitedUIAEventHandler called"); +} + +// IUnknown methods +ULONG STDMETHODCALLTYPE RateLimitedEventHandler::AddRef() { + auto refCount = InterlockedIncrement(&m_refCount); + LOG_DEBUG(L"AddRef: "<(this); + AddRef(); + return S_OK; + } else if (riid == __uuidof(IUIAutomationEventHandler)) { + *ppInterface = static_cast(this); + AddRef(); + return S_OK; + } else if (riid == __uuidof(IUIAutomationFocusChangedEventHandler)) { + *ppInterface = static_cast(this); + AddRef(); + return S_OK; + } else if (riid == __uuidof(IUIAutomationPropertyChangedEventHandler)) { + *ppInterface = static_cast(this); + AddRef(); + return S_OK; + } else if (riid == __uuidof(IUIAutomationNotificationEventHandler)) { + *ppInterface = static_cast(this); + AddRef(); + return S_OK; + } else if (riid == __uuidof(IUIAutomationActiveTextPositionChangedEventHandler)) { + *ppInterface = static_cast(this); + AddRef(); + return S_OK; + } + *ppInterface = nullptr; + return E_NOINTERFACE; +} + +// IUIAutomationEventHandler method +HRESULT STDMETHODCALLTYPE RateLimitedEventHandler::HandleAutomationEvent(IUIAutomationElement* sender, EVENTID eventID) { + LOG_DEBUG(L"RateLimitedUIAEventHandler::HandleAutomationEvent called"); + LOG_DEBUG(L"Queuing automationEvent for eventID "<(sender, eventID); +} + +// IUIAutomationFocusEventHandler method +HRESULT STDMETHODCALLTYPE RateLimitedEventHandler::HandleFocusChangedEvent(IUIAutomationElement* sender) { + LOG_DEBUG(L"RateLimitedUIAEventHandler::HandleFocusChangedEvent called"); + return queueEvent(sender); +} + +// IUIAutomationPropertyChangedEventHandler method +HRESULT STDMETHODCALLTYPE RateLimitedEventHandler::HandlePropertyChangedEvent(IUIAutomationElement* sender, PROPERTYID propertyID, VARIANT newValue) { + LOG_DEBUG(L"RateLimitedUIAEventHandler::HandlePropertyChangedEvent called"); + return queueEvent(sender, propertyID, newValue); +} + +// IUIAutomationNotificationEventHandler method +HRESULT STDMETHODCALLTYPE RateLimitedEventHandler::HandleNotificationEvent(IUIAutomationElement* sender, NotificationKind notificationKind, NotificationProcessing notificationProcessing, BSTR displayString, BSTR activityID) { + LOG_DEBUG(L"RateLimitedUIAEventHandler::HandleNotificationEvent called"); + return queueEvent(sender, notificationKind, notificationProcessing, displayString, activityID); +} + +// IUIAutomationActiveTextPositionchangedEventHandler method +HRESULT STDMETHODCALLTYPE RateLimitedEventHandler::HandleActiveTextPositionChangedEvent(IUIAutomationElement* sender, IUIAutomationTextRange* range) { + LOG_DEBUG(L"RateLimitedUIAEventHandler::HandleActiveTextPositionChangedEvent called"); + return queueEvent(sender, range); +} + +void RateLimitedEventHandler::flushEvents() { + LOG_DEBUG(L"RateLimitedEventHandler::flushEvents called"); + decltype(m_eventRecords) eventRecordsCopy; + decltype(m_eventRecordsByKey) eventRecordsByKeyCopy; + { std::lock_guard lock{m_mtx}; + eventRecordsCopy.swap(m_eventRecords); + eventRecordsByKeyCopy.swap(m_eventRecordsByKey); + } // m_mtx released here. + // Emit events + LOG_DEBUG(L"RateLimitedUIAEventHandler::flusherThreadFunc: Emitting events..."); + for(const auto& recordVar: eventRecordsCopy) { + std::visit([this](const auto& record) { + this->emitEvent(record); + }, recordVar); + } + LOG_DEBUG(L"RateLimitedUIAEventHandler::flusherThreadFunc: Done emitting events"); +} diff --git a/nvdaHelper/local/UIAEventLimiter/rateLimitedEventHandler.h b/nvdaHelper/local/UIAEventLimiter/rateLimitedEventHandler.h new file mode 100644 index 00000000000..1930829a4f8 --- /dev/null +++ b/nvdaHelper/local/UIAEventLimiter/rateLimitedEventHandler.h @@ -0,0 +1,85 @@ +/* +This file is a part of the NVDA project. +URL: http://github.com/nvaccess/nvda/ +Copyright 2023 NV Access Limited. + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License version 2.0, as published by + the Free Software Foundation. + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +This license can be found at: +http://www.gnu.org/licenses/old-licenses/gpl-2.0.html +*/ + +#include +#include +#include +#include +#include +#include "eventRecord.h" + + +// @brief a class that listens for various UI Automation events, +// stores them in in an internal queue (removing any duplicates), +// and sends them onto an existing UI Automation event handler in a separate thread. +// This ensures that UI Automation core is never blocked while sending events to this class. +class RateLimitedEventHandler: public IUIAutomationEventHandler, public IUIAutomationFocusChangedEventHandler, public IUIAutomationPropertyChangedEventHandler, public IUIAutomationNotificationEventHandler, public IUIAutomationActiveTextPositionChangedEventHandler { +private: + unsigned long m_refCount = 1; + CComQIPtr m_pExistingAutomationEventHandler; + CComQIPtr m_pExistingFocusChangedEventHandler; + CComQIPtr m_pExistingPropertyChangedEventHandler; + CComQIPtr m_pExistingNotificationEventHandler; + CComQIPtr m_pExistingActiveTextPositionChangedEventHandler; + std::list m_eventRecords; + std::map, std::pair> m_eventRecordsByKey; + bool m_needsFlush = false; + std::mutex m_mtx; + std::condition_variable m_flushConditionVar; + std::jthread m_flusherThread; + + /// @brief a thread function that wakes and flushes the event queue when one or more events have been added. + /// @param stopToken used to check if the thread should stop. + void flusherThreadFunc(std::stop_token stopToken); + + /// @brief a template function that queues a UI Automation event. + /// @tparam EventRecordClass the type of event record representing a UI Automation event. + /// @tparam ...EventRecordArgTypes the argument types required to construct the event record + /// @param ...args the arguments to construct the event record. + /// @return S_OK on success or a failure code otherwise. + template HRESULT queueEvent(EventRecordArgTypes&&... args); + + /// @brief Emits a UI Automation event to its existing handler. + /// @param record the event record representing the UI automation event. + /// @return S_OK on success or a failure code otherwise. + HRESULT emitEvent(const AutomationEventRecord_t& record) const; + HRESULT emitEvent(const FocusChangedEventRecord_t& record) const; + HRESULT emitEvent(const PropertyChangedEventRecord_t& record) const; + HRESULT emitEvent(const NotificationEventRecord_t& record) const; + HRESULT emitEvent(const ActiveTextPositionChangedEventRecord_t& record) const; + + /// @brief Removes all events from the queue, sending them on to the existing UI automation event handler. + void flushEvents(); + + ~RateLimitedEventHandler(); + + public: + + /// @brief class constructor. + /// @param pExistingHandler a pointer to an existing UI Automation event handler where events should be sent after they are flushed from the queue. + RateLimitedEventHandler(IUnknown* pExistingHandler); + + // IUnknown methods + ULONG STDMETHODCALLTYPE AddRef(); + ULONG STDMETHODCALLTYPE Release(); + HRESULT STDMETHODCALLTYPE QueryInterface(REFIID riid, void** ppInterface); + + // IUIAutomationEventHandler methods + HRESULT STDMETHODCALLTYPE HandleAutomationEvent(IUIAutomationElement* sender, EVENTID eventID); + HRESULT STDMETHODCALLTYPE HandleFocusChangedEvent(IUIAutomationElement* sender); + HRESULT STDMETHODCALLTYPE HandlePropertyChangedEvent(IUIAutomationElement* sender, PROPERTYID propertyID, VARIANT newValue); + HRESULT STDMETHODCALLTYPE HandleNotificationEvent(IUIAutomationElement* sender, NotificationKind notificationKind, NotificationProcessing notificationProcessing, BSTR displayString, BSTR activityID); + HRESULT STDMETHODCALLTYPE HandleActiveTextPositionChangedEvent(IUIAutomationElement* sender, IUIAutomationTextRange* range); + +}; diff --git a/nvdaHelper/local/UIAEventLimiter/utils.cpp b/nvdaHelper/local/UIAEventLimiter/utils.cpp new file mode 100644 index 00000000000..ae43af60fa1 --- /dev/null +++ b/nvdaHelper/local/UIAEventLimiter/utils.cpp @@ -0,0 +1,43 @@ +/* +This file is a part of the NVDA project. +URL: http://github.com/nvaccess/nvda/ +Copyright 2023 NV Access Limited. + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License version 2.0, as published by + the Free Software Foundation. + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +This license can be found at: +http://www.gnu.org/licenses/old-licenses/gpl-2.0.html +*/ + +#include +#include +#include +#include "utils.h" + +std::vector SafeArrayToVector(SAFEARRAY* pSafeArray) { + std::vector vec; + int* data; + HRESULT hr = SafeArrayAccessData(pSafeArray, (void**)&data); + if (SUCCEEDED(hr)) { + LONG lowerBound, upperBound; + SafeArrayGetLBound(pSafeArray, 1, &lowerBound); + SafeArrayGetUBound(pSafeArray, 1, &upperBound); + vec.assign(data, data + (upperBound - lowerBound + 1)); + SafeArrayUnaccessData(pSafeArray); + } + return vec; +} + +std::vector getRuntimeIDFromElement(IUIAutomationElement* pElement) { + SAFEARRAY* runtimeIdArray; + HRESULT hr = pElement->GetRuntimeId(&runtimeIdArray); + if (FAILED(hr)) { + return {}; + } + std::vector runtimeID = SafeArrayToVector(runtimeIdArray); + SafeArrayDestroy(runtimeIdArray); + return runtimeID; +} diff --git a/nvdaHelper/local/UIAEventLimiter/utils.h b/nvdaHelper/local/UIAEventLimiter/utils.h new file mode 100644 index 00000000000..2e55486e627 --- /dev/null +++ b/nvdaHelper/local/UIAEventLimiter/utils.h @@ -0,0 +1,43 @@ +/* +This file is a part of the NVDA project. +URL: http://github.com/nvaccess/nvda/ +Copyright 2023 NV Access Limited. + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License version 2.0, as published by + the Free Software Foundation. + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +This license can be found at: +http://www.gnu.org/licenses/old-licenses/gpl-2.0.html +*/ + +#include +#include +#include +#include +#include +#include +#include + +/// @brief creates a vector of ints from a SAFEARRAY. +/// @param pSafeArray +/// @return the vector of ints. +std::vector SafeArrayToVector(SAFEARRAY* pSafeArray); + +/// @brief Fetches the runtimeID from a given uI Automation element. +/// @param pElement the UI Automation element whose runtime ID should be fetched. +/// @return the runtime ID from the element. +std::vector getRuntimeIDFromElement(IUIAutomationElement* pElement); + +// @brief a helper template function for the supports_alternative concept. +template +constexpr bool supports_alternative_impl(std::index_sequence) { + return (std::same_as> || ...); +} + +// @brief a concept that checks if a given type can be held by a given variant type. +// @tparam T the type to check +// @tparam V the variant type to check +template +concept supports_alternative = supports_alternative_impl(std::make_index_sequence>{}); diff --git a/nvdaHelper/local/nvdaHelperLocal.def b/nvdaHelper/local/nvdaHelperLocal.def index e64def21c21..c4e3a710137 100644 --- a/nvdaHelper/local/nvdaHelperLocal.def +++ b/nvdaHelper/local/nvdaHelperLocal.def @@ -67,6 +67,8 @@ EXPORTS getOleClipboardText _nvdaControllerInternal_reportLiveRegion _nvdaControllerInternal_openConfigDirectory + rateLimitedUIAEventHandler_create + rateLimitedUIAEventHandler_flush wasPlay_create wasPlay_destroy wasPlay_open diff --git a/nvdaHelper/local/sconscript b/nvdaHelper/local/sconscript index 4ff5a22e66b..94099b0e492 100644 --- a/nvdaHelper/local/sconscript +++ b/nvdaHelper/local/sconscript @@ -89,6 +89,9 @@ localLib=env.SharedLibrary( "nvdaHelperLocal.def", "textUtils.cpp", "UIAUtils.cpp", + "UIAEventLimiter/api.cpp", + "UIAEventLimiter/rateLimitedEventHandler.cpp", + "UIAEventLimiter/utils.cpp", "mixer.cpp", "oleUtils.cpp", "wasapi.cpp", diff --git a/source/UIAHandler/__init__.py b/source/UIAHandler/__init__.py index 607528c8230..c8c811f152f 100644 --- a/source/UIAHandler/__init__.py +++ b/source/UIAHandler/__init__.py @@ -9,6 +9,9 @@ from ctypes import ( oledll, windll, + POINTER, + CFUNCTYPE, + c_voidp, ) import comtypes.client @@ -19,6 +22,7 @@ byref, CLSCTX_INPROC_SERVER, CoCreateInstance, + IUnknown, ) import threading @@ -46,6 +50,8 @@ from typing import Dict from queue import Queue import aria +import core +import NVDAHelper from . import remote as UIARemote @@ -442,7 +448,7 @@ def terminate(self): ) ) self.MTAThreadQueue.put_nowait(None) - #Wait for the MTA thread to die (while still message pumping) + # Wait for the MTA thread to die (while still message pumping) if windll.user32.MsgWaitForMultipleObjects(1,byref(MTAThreadHandle),False,200,0)!=0: log.debugWarning("Timeout or error while waiting for UIAHandler MTA thread") windll.kernel32.CloseHandle(MTAThreadHandle) @@ -507,6 +513,11 @@ def MTAThreadFunc(self): self.rootElement=self.clientObject.getRootElementBuildCache(self.baseCacheRequest) self.reservedNotSupportedValue=self.clientObject.ReservedNotSupportedValue self.ReservedMixedAttributeValue=self.clientObject.ReservedMixedAttributeValue + self.pRateLimitedEventHandler = POINTER(IUnknown)() + NVDAHelper.localLib.rateLimitedUIAEventHandler_create( + self._com_pointers_[IUnknown._iid_], + byref(self.pRateLimitedEventHandler) + ) if utils._shouldSelectivelyRegister(): self._createLocalEventHandlerGroup() self._registerGlobalEventHandlers() @@ -528,7 +539,7 @@ def MTAThreadFunc(self): self.clientObject.RemoveAllEventHandlers() def _registerGlobalEventHandlers(self): - self.clientObject.AddFocusChangedEventHandler(self.baseCacheRequest, self) + self.clientObject.AddFocusChangedEventHandler(self.baseCacheRequest, self.pRateLimitedEventHandler) if isinstance(self.clientObject, UIA.IUIAutomation6): self.globalEventHandlerGroup = self.clientObject.CreateEventHandlerGroup() else: @@ -536,7 +547,7 @@ def _registerGlobalEventHandlers(self): self.globalEventHandlerGroup.AddPropertyChangedEventHandler( UIA.TreeScope_Subtree, self.baseCacheRequest, - self, + self.pRateLimitedEventHandler, *self.clientObject.IntSafeArrayToNativeArray( globalEventHandlerGroupUIAPropertyIds if utils._shouldSelectivelyRegister() @@ -552,7 +563,7 @@ def _registerGlobalEventHandlers(self): eventId, UIA.TreeScope_Subtree, self.baseCacheRequest, - self + self.pRateLimitedEventHandler ) if ( not utils._shouldSelectivelyRegister() @@ -563,20 +574,20 @@ def _registerGlobalEventHandlers(self): UIA.UIA_Text_TextChangedEventId, UIA.TreeScope_Subtree, self.baseCacheRequest, - self + self.pRateLimitedEventHandler ) # #7984: add support for notification event (IUIAutomation5, part of Windows 10 build 16299 and later). if isinstance(self.clientObject, UIA.IUIAutomation5): self.globalEventHandlerGroup.AddNotificationEventHandler( UIA.TreeScope_Subtree, self.baseCacheRequest, - self + self.pRateLimitedEventHandler ) if isinstance(self.clientObject, UIA.IUIAutomation6): self.globalEventHandlerGroup.AddActiveTextPositionChangedEventHandler( UIA.TreeScope_Subtree, self.baseCacheRequest, - self + self.pRateLimitedEventHandler ) self.addEventHandlerGroup(self.rootElement, self.globalEventHandlerGroup) @@ -590,13 +601,13 @@ def _createLocalEventHandlerGroup(self): self.localEventHandlerGroup.AddPropertyChangedEventHandler( UIA.TreeScope_Ancestors | UIA.TreeScope_Element, self.baseCacheRequest, - self, + self.pRateLimitedEventHandler, *self.clientObject.IntSafeArrayToNativeArray(localEventHandlerGroupUIAPropertyIds) ) self.localEventHandlerGroupWithTextChanges.AddPropertyChangedEventHandler( UIA.TreeScope_Ancestors | UIA.TreeScope_Element, self.baseCacheRequest, - self, + self.pRateLimitedEventHandler, *self.clientObject.IntSafeArrayToNativeArray(localEventHandlerGroupUIAPropertyIds) ) for eventId in localEventHandlerGroupUIAEventIds: @@ -604,19 +615,19 @@ def _createLocalEventHandlerGroup(self): eventId, UIA.TreeScope_Ancestors | UIA.TreeScope_Element, self.baseCacheRequest, - self + self.pRateLimitedEventHandler ) self.localEventHandlerGroupWithTextChanges.AddAutomationEventHandler( eventId, UIA.TreeScope_Ancestors | UIA.TreeScope_Element, self.baseCacheRequest, - self + self.pRateLimitedEventHandler ) self.localEventHandlerGroupWithTextChanges.AddAutomationEventHandler( UIA.UIA_Text_TextChangedEventId, UIA.TreeScope_Ancestors | UIA.TreeScope_Element, self.baseCacheRequest, - self + self.pRateLimitedEventHandler ) def addEventHandlerGroup(self, element, eventHandlerGroup): @@ -730,6 +741,7 @@ def IUIAutomationEventHandler_HandleAutomationEvent(self,sender,eventID): if _isDebug(): log.debugWarning(f"HandleAutomationEvent: Don't know how to handle event {eventID}") return + obj = None focus = api.getFocusObject() import NVDAObjects.UIA if ( @@ -737,38 +749,48 @@ def IUIAutomationEventHandler_HandleAutomationEvent(self,sender,eventID): and self.clientObject.compareElements(focus.UIAElement, sender) ): if _isDebug(): - log.debug("handleAutomationEvent: element matches focus") - pass + log.debug( + "handleAutomationEvent: element matches focus. " + f"Redirecting event to focus NVDAObject {focus}" + ) + obj = focus elif not self.isNativeUIAElement(sender): if _isDebug(): log.debug( f"HandleAutomationEvent: Ignoring event {NVDAEventName} for non native element" ) return - window = self.getNearestWindowHandle(sender) - if window and not eventHandler.shouldAcceptEvent(NVDAEventName, windowHandle=window): + window = obj.windowHandle if obj else self.getNearestWindowHandle(sender) + if window: if _isDebug(): log.debug( - f"HandleAutomationEvent: Ignoring event {NVDAEventName} for shouldAcceptEvent=False" - ) - return - try: - obj = NVDAObjects.UIA.UIA(UIAElement=sender) - except Exception: - if _isDebug(): - log.debugWarning( - f"HandleAutomationEvent: Exception while creating object for event {NVDAEventName}", - exc_info=True + f"Checking if should accept NVDA event {NVDAEventName} " + f"with window {self.getWindowHandleDebugString(window)}" ) - return + if not eventHandler.shouldAcceptEvent(NVDAEventName, windowHandle=window): + if _isDebug(): + log.debug( + f"HandleAutomationEvent: Ignoring event {NVDAEventName} for shouldAcceptEvent=False" + ) + return if not obj: - if _isDebug(): - log.debug("handleAutomationEvent: No NVDAObject could be created") + try: + obj = NVDAObjects.UIA.UIA(windowHandle=window, UIAElement=sender) + except Exception: + if _isDebug(): + log.debugWarning( + f"HandleAutomationEvent: Exception while creating object for event {NVDAEventName}", + exc_info=True + ) return - if _isDebug(): - log.debug( - f"handleAutomationEvent: created object {obj} " - ) + if not obj: + if _isDebug(): + log.debug(f"handleAutomationEvent: No NVDAObject could be created") + return + if _isDebug(): + log.debug( + f"handleAutomationEvent: created object {obj} " + ) if ( (NVDAEventName == "gainFocus" and not obj.shouldAllowUIAFocusEvent) or (NVDAEventName=="liveRegionChange" and not obj._shouldAllowUIALiveRegionChangeEvent) @@ -779,10 +801,6 @@ def IUIAutomationEventHandler_HandleAutomationEvent(self,sender,eventID): f"Ignoring event {NVDAEventName} because ignored by object itself" ) return - if obj==focus: - if _isDebug(): - log.debug("handleAutomationEvent: redirecting event to focus") - obj=focus if _isDebug(): log.debug( f"handleAutomationEvent: queuing NVDA event {NVDAEventName} " @@ -847,7 +865,7 @@ def IUIAutomationFocusChangedEventHandler_HandleFocusChangedEvent(self,sender): ) return try: - obj = NVDAObjects.UIA.UIA(UIAElement=sender) + obj = NVDAObjects.UIA.UIA(windowHandle=window, UIAElement=sender) except Exception: if _isDebug(): log.debugWarning( @@ -909,6 +927,7 @@ def IUIAutomationPropertyChangedEventHandler_HandlePropertyChangedEvent(self,sen if _isDebug(): log.debugWarning(f"HandlePropertyChangedEvent: Don't know how to handle property {propertyId}") return + obj = None focus = api.getFocusObject() import NVDAObjects.UIA if ( @@ -916,42 +935,48 @@ def IUIAutomationPropertyChangedEventHandler_HandlePropertyChangedEvent(self,sen and self.clientObject.compareElements(focus.UIAElement, sender) ): if _isDebug(): - log.debug("propertyChange event is for focus") - pass + log.debug( + "propertyChange event is for focus. " + f"Redirecting event to focus NVDAObject {focus}" + ) + obj = focus elif not self.isNativeUIAElement(sender): if _isDebug(): log.debug( f"HandlePropertyChangedEvent: Ignoring event {NVDAEventName} for non native element" ) return - window = self.getNearestWindowHandle(sender) - if window and not eventHandler.shouldAcceptEvent(NVDAEventName, windowHandle=window): + window = obj.windowHandle if obj else self.getNearestWindowHandle(sender) + if window: if _isDebug(): log.debug( - f"HandlePropertyChangedEvent: Ignoring event {NVDAEventName} for shouldAcceptEvent=False" + f"Checking if should accept NVDA event {NVDAEventName} " + f"with window {self.getWindowHandleDebugString(window)}" ) - return - try: - obj = NVDAObjects.UIA.UIA(UIAElement=sender) - except Exception: - if _isDebug(): - log.debugWarning( - f"HandlePropertyChangedEvent: Exception while creating object for event {NVDAEventName}", - exc_info=True - ) - return + if not eventHandler.shouldAcceptEvent(NVDAEventName, windowHandle=window): + if _isDebug(): + log.debug( + f"HandlePropertyChangedEvent: Ignoring event {NVDAEventName} for shouldAcceptEvent=False" + ) + return if not obj: + try: + obj = NVDAObjects.UIA.UIA(windowHandle=window, UIAElement=sender) + except Exception: + if _isDebug(): + log.debugWarning( + f"HandlePropertyChangedEvent: Exception while creating object for event {NVDAEventName}", + exc_info=True + ) + return + if not obj: + if _isDebug(): + log.debug(f"HandlePropertyChangedEvent: Ignoring event {NVDAEventName} because no object") + return if _isDebug(): - log.debug(f"HandlePropertyChangedEvent: Ignoring event {NVDAEventName} because no object") - return - if _isDebug(): - log.debug( - f"handlePropertyChangeEvent: created object {obj} " - ) - if obj==focus: - if _isDebug(): - log.debug("handlePropertyChangeEvent: redirecting to focus") - obj=focus + log.debug( + f"handlePropertyChangeEvent: created object {obj} " + ) if _isDebug(): log.debug( f"handlePropertyChangeEvent: queuing NVDA {NVDAEventName} event " diff --git a/source/core.py b/source/core.py index 724241b59b2..51bfe9721b3 100644 --- a/source/core.py +++ b/source/core.py @@ -477,6 +477,9 @@ def _doLoseFocus(): log.exception("Lose focus error") +messageWindow = None + + def main(): """NVDA's core main loop. This initializes all modules such as audio, IAccessible, keyboard, mouse, and GUI. @@ -621,8 +624,8 @@ def onEndSession(evt): from winAPI.messageWindow import _MessageWindow import versionInfo + global messageWindow messageWindow = _MessageWindow(versionInfo.name) - # initialize wxpython localization support wxLocaleObj = wx.Locale() wxLang = getWxLangOrNone()