diff --git a/chromeos/services/libassistant/grpc/assistant_client.h b/chromeos/services/libassistant/grpc/assistant_client.h index 3abc1d62e55d87..7b8cd88ea0bd4b 100644 --- a/chromeos/services/libassistant/grpc/assistant_client.h +++ b/chromeos/services/libassistant/grpc/assistant_client.h @@ -24,6 +24,7 @@ class OnAssistantDisplayEventRequest; class OnConversationStateEventRequest; class OnDeviceStateEventRequest; class OnDisplayRequestRequest; +class OnMediaActionFallbackEventRequest; class OnSpeakerIdEnrollmentEventRequest; class StartSpeakerIdEnrollmentRequest; class UpdateAssistantSettingsResponse; @@ -78,6 +79,8 @@ class AssistantClient { // Media: using MediaStatus = ::assistant::api::events::DeviceState::MediaStatus; using OnDeviceStateEventRequest = ::assistant::api::OnDeviceStateEventRequest; + using OnMediaActionFallbackEventRequest = + ::assistant::api::OnMediaActionFallbackEventRequest; // Conversation: using OnConversationStateEventRequest = @@ -137,9 +140,10 @@ class AssistantClient { // Sets the current media status of media playing outside of libassistant. // Setting external state will stop any internally playing media. virtual void SetExternalPlaybackState(const MediaStatus& status_proto) = 0; - virtual void AddDeviceStateEventObserver( GrpcServicesObserver* observer) = 0; + virtual void AddMediaActionFallbackEventObserver( + GrpcServicesObserver* observer) = 0; // Conversation methods. virtual void SendVoicelessInteraction( diff --git a/chromeos/services/libassistant/grpc/assistant_client_impl.cc b/chromeos/services/libassistant/grpc/assistant_client_impl.cc index b3c354458f1870..d8b176df044e19 100644 --- a/chromeos/services/libassistant/grpc/assistant_client_impl.cc +++ b/chromeos/services/libassistant/grpc/assistant_client_impl.cc @@ -175,6 +175,11 @@ void AssistantClientImpl::AddDeviceStateEventObserver( grpc_services_.AddDeviceStateEventObserver(observer); } +void AssistantClientImpl::AddMediaActionFallbackEventObserver( + GrpcServicesObserver* observer) { + grpc_services_.AddMediaActionFallbackEventObserver(observer); +} + void AssistantClientImpl::SendVoicelessInteraction( const ::assistant::api::Interaction& interaction, const std::string& description, diff --git a/chromeos/services/libassistant/grpc/assistant_client_impl.h b/chromeos/services/libassistant/grpc/assistant_client_impl.h index de2ebe17667cf9..6e2793f79a1711 100644 --- a/chromeos/services/libassistant/grpc/assistant_client_impl.h +++ b/chromeos/services/libassistant/grpc/assistant_client_impl.h @@ -54,6 +54,9 @@ class AssistantClientImpl : public AssistantClientV1 { GrpcServicesObserver* observer) override; void AddDeviceStateEventObserver( GrpcServicesObserver* observer) override; + void AddMediaActionFallbackEventObserver( + GrpcServicesObserver* observer) + override; void SendVoicelessInteraction( const ::assistant::api::Interaction& interaction, const std::string& description, diff --git a/chromeos/services/libassistant/grpc/assistant_client_v1.cc b/chromeos/services/libassistant/grpc/assistant_client_v1.cc index 533a72c747f80c..a757da88f69e68 100644 --- a/chromeos/services/libassistant/grpc/assistant_client_v1.cc +++ b/chromeos/services/libassistant/grpc/assistant_client_v1.cc @@ -29,6 +29,7 @@ #include "chromeos/assistant/internal/proto/shared/proto/v2/speaker_id_enrollment_interface.pb.h" #include "chromeos/services/assistant/public/cpp/features.h" #include "chromeos/services/libassistant/callback_utils.h" +#include "chromeos/services/libassistant/grpc/assistant_client.h" #include "chromeos/services/libassistant/grpc/utils/media_status_utils.h" #include "chromeos/services/libassistant/grpc/utils/settings_utils.h" #include "chromeos/services/libassistant/grpc/utils/timer_utils.h" @@ -443,6 +444,17 @@ void AssistantClientV1::AddDeviceStateEventObserver( device_state_event_observer_list_.AddObserver(observer); } +void AssistantClientV1::AddMediaActionFallbackEventObserver( + GrpcServicesObserver* observer) { + media_action_fallback_event_observer_list_.AddObserver(observer); + + // Register handler for media actions. + auto callback = base::BindRepeating(&AssistantClientV1::HandleMediaAction, + weak_factory_.GetWeakPtr()); + assistant_manager_internal()->RegisterFallbackMediaHandler( + ToStdFunctionRepeating(BindToCurrentSequenceRepeating(callback))); +} + void AssistantClientV1::SendVoicelessInteraction( const ::assistant::api::Interaction& interaction, const std::string& description, @@ -536,6 +548,19 @@ void AssistantClientV1::AddMediaManagerListener() { media_manager_listener_.get()); } +void AssistantClientV1::HandleMediaAction( + const std::string& action_name, + const std::string& media_action_args_proto) { + OnMediaActionFallbackEventRequest request; + auto* media_action = request.mutable_event()->mutable_on_media_action_event(); + media_action->set_action_name(action_name); + media_action->set_action_args(media_action_args_proto); + + for (auto& observer : media_action_fallback_event_observer_list_) { + observer.OnGrpcMessage(request); + } +} + void AssistantClientV1::NotifyConversationStateEvent( const OnConversationStateEventRequest& request) { for (auto& observer : conversation_state_event_observer_list_) { diff --git a/chromeos/services/libassistant/grpc/assistant_client_v1.h b/chromeos/services/libassistant/grpc/assistant_client_v1.h index 693ea5b9335a63..e736ff1cef4d9f 100644 --- a/chromeos/services/libassistant/grpc/assistant_client_v1.h +++ b/chromeos/services/libassistant/grpc/assistant_client_v1.h @@ -57,6 +57,9 @@ class AssistantClientV1 : public AssistantClient { void SetExternalPlaybackState(const MediaStatus& status_proto) override; void AddDeviceStateEventObserver( GrpcServicesObserver* observer) override; + void AddMediaActionFallbackEventObserver( + GrpcServicesObserver* observer) + override; void SendVoicelessInteraction( const ::assistant::api::Interaction& interaction, const std::string& description, @@ -108,6 +111,8 @@ class AssistantClientV1 : public AssistantClient { class AssistantManagerDelegateImpl; void AddMediaManagerListener(); + void HandleMediaAction(const std::string& action_name, + const std::string& media_action_args_proto); void NotifyConversationStateEvent( const OnConversationStateEventRequest& request); @@ -141,6 +146,9 @@ class AssistantClientV1 : public AssistantClient { base::ObserverList> device_state_event_observer_list_; + base::ObserverList> + media_action_fallback_event_observer_list_; + base::ObserverList< GrpcServicesObserver<::assistant::api::OnAlarmTimerEventRequest>> timer_event_observer_list_; diff --git a/chromeos/services/libassistant/grpc/external_services/event_handler_driver.cc b/chromeos/services/libassistant/grpc/external_services/event_handler_driver.cc index efabe8d6ba62f8..1b604c0e8ffbc4 100644 --- a/chromeos/services/libassistant/grpc/external_services/event_handler_driver.cc +++ b/chromeos/services/libassistant/grpc/external_services/event_handler_driver.cc @@ -16,6 +16,7 @@ constexpr char kAlarmTimerEventName[] = "AlarmTimerEvent"; constexpr char kAssistantDisplayEventName[] = "AssistantDisplayEvent"; constexpr char kConversationStateEventName[] = "ConversationStateEvent"; constexpr char kDeviceStateEventName[] = "DeviceStateEvent"; +constexpr char kMediaActionFallbackEventName[] = "MediaActionFallbackEvent"; constexpr char kHandlerMethodName[] = "OnEventFromLibas"; template @@ -75,5 +76,16 @@ CreateRegistrationRequest<::assistant::api::DeviceStateEventHandlerInterface>( return request; } +template <> +::assistant::api::RegisterEventHandlerRequest CreateRegistrationRequest< + ::assistant::api::MediaActionFallbackEventHandlerInterface>( + const std::string& assistant_service_address) { + ::assistant::api::RegisterEventHandlerRequest request; + PopulateRequest(assistant_service_address, kMediaActionFallbackEventName, + &request, + request.mutable_media_action_fallback_events_to_handle()); + return request; +} + } // namespace libassistant } // namespace chromeos diff --git a/chromeos/services/libassistant/grpc/external_services/grpc_services_initializer.cc b/chromeos/services/libassistant/grpc/external_services/grpc_services_initializer.cc index 2ae8909bd9878e..028c4bb320cef3 100644 --- a/chromeos/services/libassistant/grpc/external_services/grpc_services_initializer.cc +++ b/chromeos/services/libassistant/grpc/external_services/grpc_services_initializer.cc @@ -98,6 +98,12 @@ void GrpcServicesInitializer::AddDeviceStateEventObserver( device_state_event_handler_driver_->AddObserver(observer); } +void GrpcServicesInitializer::AddMediaActionFallbackEventObserver( + GrpcServicesObserver<::assistant::api::OnMediaActionFallbackEventRequest>* + observer) { + media_action_fallback_event_handler_driver_->AddObserver(observer); +} + ActionService* GrpcServicesInitializer::GetActionService() { return action_handler_driver_.get(); } @@ -140,6 +146,14 @@ void GrpcServicesInitializer::InitDrivers(grpc::ServerBuilder* server_builder) { EventHandlerDriver<::assistant::api::DeviceStateEventHandlerInterface>>( &server_builder_, libassistant_client_.get(), assistant_service_address_); service_drivers_.emplace_back(device_state_event_handler_driver_.get()); + + media_action_fallback_event_handler_driver_ = + std::make_unique>( + &server_builder_, libassistant_client_.get(), + assistant_service_address_); + service_drivers_.emplace_back( + media_action_fallback_event_handler_driver_.get()); } void GrpcServicesInitializer::InitLibassistGrpcClient() { @@ -172,6 +186,7 @@ void GrpcServicesInitializer::RegisterEventHandlers() { assistant_display_event_handler_driver_->StartRegistration(); conversation_state_event_handler_driver_->StartRegistration(); device_state_event_handler_driver_->StartRegistration(); + media_action_fallback_event_handler_driver_->StartRegistration(); } } // namespace libassistant diff --git a/chromeos/services/libassistant/grpc/external_services/grpc_services_initializer.h b/chromeos/services/libassistant/grpc/external_services/grpc_services_initializer.h index 1f8d7f108a8ec6..2a5d2fdf2cb6bb 100644 --- a/chromeos/services/libassistant/grpc/external_services/grpc_services_initializer.h +++ b/chromeos/services/libassistant/grpc/external_services/grpc_services_initializer.h @@ -24,6 +24,7 @@ class AlarmTimerEventHandlerInterface; class AssistantDisplayEventHandlerInterface; class ConversationStateEventHandlerInterface; class DeviceStateEventHandlerInterface; +class MediaActionFallbackEventHandlerInterface; } // namespace api } // namespace assistant @@ -63,6 +64,9 @@ class GrpcServicesInitializer : public ServicesInitializerBase { void AddDeviceStateEventObserver( GrpcServicesObserver<::assistant::api::OnDeviceStateEventRequest>* observer); + void AddMediaActionFallbackEventObserver( + GrpcServicesObserver<::assistant::api::OnMediaActionFallbackEventRequest>* + observer); ActionService* GetActionService(); @@ -133,6 +137,10 @@ class GrpcServicesInitializer : public ServicesInitializerBase { std::unique_ptr< EventHandlerDriver<::assistant::api::DeviceStateEventHandlerInterface>> device_state_event_handler_driver_; + + std::unique_ptr> + media_action_fallback_event_handler_driver_; }; } // namespace libassistant diff --git a/chromeos/services/libassistant/media_controller.cc b/chromeos/services/libassistant/media_controller.cc index 5d00f6c228c2a7..6de0ede3c31ebd 100644 --- a/chromeos/services/libassistant/media_controller.cc +++ b/chromeos/services/libassistant/media_controller.cc @@ -97,14 +97,15 @@ std::string GetWebUrlFromMediaArgs(const std::string& play_media_args_proto) { } // namespace -class MediaController::DeviceStateEventObserver - : public GrpcServicesObserver<::assistant::api::OnDeviceStateEventRequest> { +class MediaController::GrpcEventsObserver + : public GrpcServicesObserver<::assistant::api::OnDeviceStateEventRequest>, + public GrpcServicesObserver< + ::assistant::api::OnMediaActionFallbackEventRequest> { public: - explicit DeviceStateEventObserver(MediaController* parent) - : parent_(parent) {} - DeviceStateEventObserver(const DeviceStateEventObserver&) = delete; - DeviceStateEventObserver& operator=(const DeviceStateEventObserver&) = delete; - ~DeviceStateEventObserver() override = default; + explicit GrpcEventsObserver(MediaController* parent) : parent_(parent) {} + GrpcEventsObserver(const GrpcEventsObserver&) = delete; + GrpcEventsObserver& operator=(const GrpcEventsObserver&) = delete; + ~GrpcEventsObserver() override = default; // GrpcServicesObserver: // Invoked when a device state event has been received. @@ -123,37 +124,20 @@ class MediaController::DeviceStateEventObserver ConvertMediaStatusToMojomFromV2(new_state.media_status())); } - private: - mojom::MediaDelegate& delegate() { return *parent_->delegate_; } - - MediaController* const parent_; -}; + // Invoked when a media action fallack event has been received. + void OnGrpcMessage(const ::assistant::api::OnMediaActionFallbackEventRequest& + request) override { + if (!request.event().has_on_media_action_event()) + return; -class MediaController::LibassistantMediaHandler { - public: - LibassistantMediaHandler( - MediaController* parent, - assistant_client::AssistantManagerInternal* assistant_manager_internal) - : parent_(parent), - mojom_task_runner_(base::SequencedTaskRunnerHandle::Get()) { -#if !BUILDFLAG(IS_PREBUILT_LIBASSISTANT) - // Register handler for media actions. - assistant_manager_internal->RegisterFallbackMediaHandler( - [this](std::string action_name, std::string media_action_args_proto) { - HandleMediaAction(action_name, media_action_args_proto); - }); -#endif // !BUILDFLAG(IS_PREBUILT_LIBASSISTANT) + auto media_action_event = request.event().on_media_action_event(); + HandleMediaAction(media_action_event.action_name(), + media_action_event.action_args()); } - LibassistantMediaHandler(const LibassistantMediaHandler&) = delete; - LibassistantMediaHandler& operator=(const LibassistantMediaHandler&) = delete; - ~LibassistantMediaHandler() = default; private: - // Called from the Libassistant thread. void HandleMediaAction(const std::string& action_name, const std::string& media_action_args_proto) { - ENSURE_MOJOM_THREAD(&LibassistantMediaHandler::HandleMediaAction, - action_name, media_action_args_proto); if (action_name == kPlayMediaClientOp) OnPlayMedia(media_action_args_proto); else @@ -229,13 +213,10 @@ class MediaController::LibassistantMediaHandler { mojom::MediaDelegate& delegate() { return *parent_->delegate_; } MediaController* const parent_; - scoped_refptr mojom_task_runner_; - base::WeakPtrFactory weak_factory_{this}; }; MediaController::MediaController() - : device_state_event_observer_( - std::make_unique(this)) {} + : events_observer_(std::make_unique(this)) {} MediaController::~MediaController() = default; @@ -271,24 +252,9 @@ void MediaController::SetExternalPlaybackState(mojom::MediaStatePtr state) { void MediaController::OnAssistantClientRunning( AssistantClient* assistant_client) { assistant_client_ = assistant_client; - - handler_ = std::make_unique( - this, assistant_client->assistant_manager_internal()); - - // |device_state_event_observer_| outlives |assistant_client_|. - assistant_client->AddDeviceStateEventObserver( - device_state_event_observer_.get()); -} - -void MediaController::OnDestroyingAssistantClient( - AssistantClient* assistant_client) { - assistant_client_ = nullptr; -} - -void MediaController::OnAssistantClientDestroyed() { - // Handler can only be unset after the |AssistantManagerInternal| has been - // destroyed, as |AssistantManagerInternal| will call the handler. - handler_ = nullptr; + // `events_observer_` outlives `assistant_client_`. + assistant_client->AddDeviceStateEventObserver(events_observer_.get()); + assistant_client->AddMediaActionFallbackEventObserver(events_observer_.get()); } } // namespace libassistant diff --git a/chromeos/services/libassistant/media_controller.h b/chromeos/services/libassistant/media_controller.h index 68c23458c48f27..93e414031a45ad 100644 --- a/chromeos/services/libassistant/media_controller.h +++ b/chromeos/services/libassistant/media_controller.h @@ -32,19 +32,15 @@ class MediaController : public mojom::MediaController, // AssistantClientObserver implementation: void OnAssistantClientRunning(AssistantClient* assistant_client) override; - void OnDestroyingAssistantClient(AssistantClient* assistant_client) override; - void OnAssistantClientDestroyed() override; private: - class DeviceStateEventObserver; - class LibassistantMediaHandler; + class GrpcEventsObserver; AssistantClient* assistant_client_ = nullptr; mojo::Receiver receiver_{this}; mojo::Remote delegate_; - std::unique_ptr device_state_event_observer_; - std::unique_ptr handler_; + std::unique_ptr events_observer_; }; } // namespace libassistant diff --git a/chromeos/services/libassistant/test_support/fake_assistant_client.cc b/chromeos/services/libassistant/test_support/fake_assistant_client.cc index 1ee6c8c12c1bc7..ccf3377f7587b3 100644 --- a/chromeos/services/libassistant/test_support/fake_assistant_client.cc +++ b/chromeos/services/libassistant/test_support/fake_assistant_client.cc @@ -88,6 +88,9 @@ void FakeAssistantClient::StopAssistantInteraction(bool cancel_conversation) {} void FakeAssistantClient::AddConversationStateEventObserver( GrpcServicesObserver* observer) {} +void FakeAssistantClient::AddMediaActionFallbackEventObserver( + GrpcServicesObserver* observer) {} + void FakeAssistantClient::SetInternalOptions(const std::string& locale, bool spoken_feedback_enabled) {} diff --git a/chromeos/services/libassistant/test_support/fake_assistant_client.h b/chromeos/services/libassistant/test_support/fake_assistant_client.h index f7f4126bff243d..9bfc1af7a5df44 100644 --- a/chromeos/services/libassistant/test_support/fake_assistant_client.h +++ b/chromeos/services/libassistant/test_support/fake_assistant_client.h @@ -56,6 +56,9 @@ class FakeAssistantClient : public AssistantClient { void SetExternalPlaybackState(const MediaStatus& status_proto) override; void AddDeviceStateEventObserver( GrpcServicesObserver* observer) override; + void AddMediaActionFallbackEventObserver( + GrpcServicesObserver* observer) + override; void SendVoicelessInteraction( const ::assistant::api::Interaction& interaction, const std::string& description,