Merge branch 'ptt-mic-mode' of https://github.com/inworld-ai/inworld-…

…unreal-sdk into ptt-mic-mode

InworldAI/Source/InworldAIClient/Private/InworldClient.cpp

@@ -467,20 +467,24 @@ void UInworldClient::SendSoundMessageToConversation(const FString& ConversationI
 	}
 }
 
-void UInworldClient::SendAudioSessionStart(const FString& AgentId)
+void UInworldClient::SendAudioSessionStart(const FString& AgentId, EInworldMicrophoneMode MicrophoneMode/* = EInworldMicrophoneMode::OPEN_MIC*/)
 {
 	NO_CLIENT_RETURN(void())
 	EMPTY_ARG_RETURN(AgentId, void())
 
-	Inworld::GetClient()->StartAudioSession(TCHAR_TO_UTF8(*AgentId));
+	Inworld::AudioSessionStartPayload AudioSessionStartPayload;
+	AudioSessionStartPayload.MicMode = static_cast<Inworld::AudioSessionStartPayload::MicrophoneMode>(MicrophoneMode);
+	Inworld::GetClient()->StartAudioSession(TCHAR_TO_UTF8(*AgentId), AudioSessionStartPayload);
 }
 
-void UInworldClient::SendAudioSessionStartToConversation(const FString& ConversationId)
+void UInworldClient::SendAudioSessionStartToConversation(const FString& ConversationId, EInworldMicrophoneMode MicrophoneMode/* = EInworldMicrophoneMode::OPEN_MIC*/)
 {
 	NO_CLIENT_RETURN(void())
 	EMPTY_ARG_RETURN(ConversationId, void())
 
-	Inworld::GetClient()->StartAudioSessionInConversation(TCHAR_TO_UTF8(*ConversationId));
+	Inworld::AudioSessionStartPayload AudioSessionStartPayload;
+	AudioSessionStartPayload.MicMode = static_cast<Inworld::AudioSessionStartPayload::MicrophoneMode>(MicrophoneMode);
+	Inworld::GetClient()->StartAudioSessionInConversation(TCHAR_TO_UTF8(*ConversationId), AudioSessionStartPayload);
 }
 
 void UInworldClient::SendAudioSessionStop(const FString& AgentId)

InworldAI/Source/InworldAIClient/Public/InworldClient.h

@@ -85,9 +85,9 @@ class INWORLDAICLIENT_API UInworldClient : public UObject
 	void SendSoundMessageToConversation(const FString& ConversationId, const TArray<uint8>& InputData, const TArray<uint8>& OutputData);
 
 	UFUNCTION(BlueprintCallable, Category = "Message|Audio")
-	void SendAudioSessionStart(const FString& AgentId);
+	void SendAudioSessionStart(const FString& AgentId, EInworldMicrophoneMode MicrophoneMode = EInworldMicrophoneMode::OPEN_MIC);
 	UFUNCTION(BlueprintCallable, Category = "Message|Audio")
-	void SendAudioSessionStartToConversation(const FString& ConversationId);
+	void SendAudioSessionStartToConversation(const FString& ConversationId, EInworldMicrophoneMode MicrophoneMode = EInworldMicrophoneMode::OPEN_MIC);
 
 	UFUNCTION(BlueprintCallable, Category = "Message|Audio")
 	void SendAudioSessionStop(const FString& AgentId);

InworldAI/Source/InworldAIClient/Public/InworldEnums.h

@@ -95,3 +95,11 @@ enum class EInworldConversationUpdateType : uint8
 	UPDATED = 2,
 	EVICTED = 3,
 };
+
+UENUM(BlueprintType)
+enum class EInworldMicrophoneMode : uint8
+{
+	UNKNOWN = 0 UMETA(Hidden),
+	OPEN_MIC = 1,
+	EXPECT_AUDIO_END = 2,
+};

InworldAI/Source/InworldAIIntegration/Private/InworldApi.cpp

@@ -243,12 +243,12 @@ void UInworldApiSubsystem::SendAudioMessage(const FString& AgentId, const TArray
     InworldSession->GetClient()->SendSoundMessage(AgentId, InputData, OutputData);
 }
 
-void UInworldApiSubsystem::StartAudioSession(const FString& AgentId)
+void UInworldApiSubsystem::StartAudioSession(const FString& AgentId, EInworldMicrophoneMode MicrophoneMode/* = EInworldMicrophoneMode::OPEN_MIC*/)
 {
     NO_CLIENT_RETURN(void())
     EMPTY_ARG_RETURN(AgentId, void())
 
-    InworldSession->GetClient()->SendAudioSessionStart(AgentId);
+    InworldSession->GetClient()->SendAudioSessionStart(AgentId, MicrophoneMode);
 }
 
 void UInworldApiSubsystem::StopAudioSession(const FString& AgentId)

InworldAI/Source/InworldAIIntegration/Private/InworldCharacter.cpp

@@ -121,12 +121,12 @@ void UInworldCharacter::SendNarrationEvent(const FString& Content)
 	Session->SendNarrationEvent(this, Content);
 }
 
-void UInworldCharacter::SendAudioSessionStart()
+void UInworldCharacter::SendAudioSessionStart(EInworldMicrophoneMode MicrophoneMode/* = EInworldMicrophoneMode::OPEN_MIC*/)
 {
 	NO_SESSION_RETURN(void())
 	EMPTY_ARG_RETURN(AgentInfo.AgentId, void())
 
-	Session->SendAudioSessionStart(this);
+	Session->SendAudioSessionStart(this, MicrophoneMode);
 }
 
 void UInworldCharacter::SendAudioSessionStop()

InworldAI/Source/InworldAIIntegration/Private/InworldCharacterComponent.cpp

@@ -293,11 +293,11 @@ void UInworldCharacterComponent::SendNarrationEvent(const FString& Content)
 	InworldCharacter->SendNarrationEvent(Content);
 }
 
-void UInworldCharacterComponent::StartAudioSession()
+void UInworldCharacterComponent::StartAudioSession(EInworldMicrophoneMode MicrophoneMode/* = EInworldMicrophoneMode::OPEN_MIC*/)
 {
 	NO_CHARACTER_RETURN(void())
 
-	InworldCharacter->SendAudioSessionStart();
+	InworldCharacter->SendAudioSessionStart(MicrophoneMode);
 }
 
 void UInworldCharacterComponent::StopAudioSession()

InworldAI/Source/InworldAIIntegration/Private/InworldPlayer.cpp

@@ -94,7 +94,7 @@ void UInworldPlayer::SendTriggerToConversation(const FString& Name, const TMap<F
 	Session->SendTriggerToConversation(this, Name, Params);
 }
 
-void UInworldPlayer::SendAudioSessionStartToConversation()
+void UInworldPlayer::SendAudioSessionStartToConversation(EInworldMicrophoneMode MicrophoneMode/* = EInworldMicrophoneMode::OPEN_MIC*/)
 {
 	NO_SESSION_RETURN(void())
 	EMPTY_ARG_RETURN(ConversationId, void())
@@ -104,7 +104,9 @@ void UInworldPlayer::SendAudioSessionStartToConversation()
 		return;
 	}
 	bHasAudioSession = true;
-	Session->SendAudioSessionStartToConversation(this);
+
+	MicMode = MicrophoneMode;
+	Session->SendAudioSessionStartToConversation(this, MicrophoneMode);
 }
 
 void UInworldPlayer::SendAudioSessionStopToConversation()
@@ -117,6 +119,8 @@ void UInworldPlayer::SendAudioSessionStopToConversation()
 		return;
 	}
 	bHasAudioSession = false;
+
+	MicMode = EInworldMicrophoneMode::UNKNOWN;
 	Session->SendAudioSessionStopToConversation(this);
 }
 
@@ -224,7 +228,7 @@ void UInworldPlayer::UpdateConversation()
 
 	if (bHadAudioSession && !ConversationId.IsEmpty())
 	{
-		SendAudioSessionStartToConversation();
+		SendAudioSessionStartToConversation(MicMode);
 	}
 }
 

InworldAI/Source/InworldAIIntegration/Private/InworldPlayerAudioCaptureComponent.cpp

@@ -310,7 +310,7 @@ void UInworldPlayerAudioCaptureComponent::EvaluateVoiceCapture()
             {
                 if (!InworldPlayer->HasAudioSession())
                 {
-                    InworldPlayer->SendAudioSessionStartToConversation();
+                    InworldPlayer->SendAudioSessionStartToConversation(MicMode);
                 }
             }
             else
@@ -328,13 +328,32 @@ void UInworldPlayerAudioCaptureComponent::EvaluateVoiceCapture()
                 Rep_ServerCapturingVoice();
             }
         }
+        else if (bShouldCaptureVoice && InworldPlayer->HasAudioSession() && bIsMicModeDirty)
+        {
+            InworldPlayer->SendAudioSessionStartToConversation(MicMode);
+            InworldPlayer->SendAudioSessionStopToConversation();
+        }
     }
 }
 
 void UInworldPlayerAudioCaptureComponent::ServerSetMuted_Implementation(bool bInMuted)
 {
-    bMuted = bInMuted;
-    EvaluateVoiceCapture();
+    if (bMuted != bInMuted)
+    {
+        bMuted = bInMuted;
+        EvaluateVoiceCapture();
+    }
+}
+
+void UInworldPlayerAudioCaptureComponent::ServerSetMicMode_Implementation(EInworldMicrophoneMode InMicMode)
+{
+    if (MicMode != InMicMode)
+    {
+        MicMode = InMicMode;
+        bIsMicModeDirty = true;
+        EvaluateVoiceCapture();
+        bIsMicModeDirty = false;
+    }
 }
 
 void UInworldPlayerAudioCaptureComponent::SetCaptureDeviceById(const FString& DeviceId)

InworldAI/Source/InworldAIIntegration/Private/InworldSession.cpp

@@ -347,20 +347,20 @@ void UInworldSession::SendSoundMessageToConversation(UInworldPlayer* Player, con
 	Client->SendSoundMessageToConversation(Player->GetConversationId(), InputData, OutputData);
 }
 
-void UInworldSession::SendAudioSessionStart(UInworldCharacter* Character)
+void UInworldSession::SendAudioSessionStart(UInworldCharacter* Character, EInworldMicrophoneMode MicrophoneMode/* = EInworldMicrophoneMode::OPEN_MIC*/)
 {
 	NO_CLIENT_RETURN(void())
 	INVALID_CHARACTER_RETURN(void())
 
-	Client->SendAudioSessionStart(Character->GetAgentInfo().AgentId);
+	Client->SendAudioSessionStart(Character->GetAgentInfo().AgentId, MicrophoneMode);
 }
 
-void UInworldSession::SendAudioSessionStartToConversation(UInworldPlayer* Player)
+void UInworldSession::SendAudioSessionStartToConversation(UInworldPlayer* Player, EInworldMicrophoneMode MicrophoneMode/* = EInworldMicrophoneMode::OPEN_MIC*/)
 {
 	NO_CLIENT_RETURN(void())
 	INVALID_PLAYER_RETURN(void())
 
-	Client->SendAudioSessionStartToConversation(Player->GetConversationId());
+	Client->SendAudioSessionStartToConversation(Player->GetConversationId(), MicrophoneMode);
 }
 
 void UInworldSession::SendAudioSessionStop(UInworldCharacter* Character)

InworldAI/Source/InworldAIIntegration/Public/InworldApi.h

@@ -149,7 +149,7 @@ class INWORLDAIINTEGRATION_API UInworldApiSubsystem : public UWorldSubsystem
      * call before sending audio messages
      */
     UFUNCTION(BlueprintCallable, Category = "Audio")
-    void StartAudioSession(const FString& AgentId);
+    void StartAudioSession(const FString& AgentId, EInworldMicrophoneMode MicrophoneMode = EInworldMicrophoneMode::OPEN_MIC);
 
     /**
      * Stop audio session with agent

InworldAI/Source/InworldAIIntegration/Public/InworldCharacter.h

@@ -52,7 +52,7 @@ class INWORLDAIINTEGRATION_API UInworldCharacter : public UObject
 	UFUNCTION(BlueprintCallable, Category = "Message|Narration")
 	void SendNarrationEvent(const FString& Content);
 	UFUNCTION(BlueprintCallable, Category = "Message|Audio")
-	void SendAudioSessionStart();
+	void SendAudioSessionStart(EInworldMicrophoneMode MicrophoneMode = EInworldMicrophoneMode::OPEN_MIC);
 	UFUNCTION(BlueprintCallable, Category = "Message|Audio")
 	void SendAudioSessionStop();
 	UFUNCTION(BlueprintCallable, Category = "Message|Audio")

InworldAI/Source/InworldAIIntegration/Public/InworldCharacterComponent.h

@@ -124,7 +124,7 @@ class INWORLDAIINTEGRATION_API UInworldCharacterComponent : public UActorCompone
 	void SendNarrationEvent(const FString& Content);
 
 	UFUNCTION(BlueprintCallable, Category = "Interaction")
-	void StartAudioSession();
+	void StartAudioSession(EInworldMicrophoneMode MicrophoneMode = EInworldMicrophoneMode::OPEN_MIC);
 
 	UFUNCTION(BlueprintCallable, Category = "Interaction")
 	void StopAudioSession();

InworldAI/Source/InworldAIIntegration/Public/InworldPlayer.h

@@ -50,7 +50,7 @@ class INWORLDAIINTEGRATION_API UInworldPlayer : public UObject
 	UFUNCTION(BlueprintCallable, Category = "Message|Trigger")
 	void SendTriggerToConversation(const FString& Name, const TMap<FString, FString>& Params);
 	UFUNCTION(BlueprintCallable, Category = "Message|Audio")
-	void SendAudioSessionStartToConversation();
+	void SendAudioSessionStartToConversation(EInworldMicrophoneMode MicrophoneMode = EInworldMicrophoneMode::OPEN_MIC);
 	UFUNCTION(BlueprintCallable, Category = "Message|Audio")
 	void SendAudioSessionStopToConversation();
 	UFUNCTION(BlueprintCallable, Category = "Message|Audio")
@@ -91,8 +91,8 @@ class INWORLDAIINTEGRATION_API UInworldPlayer : public UObject
 	FOnInworldPlayerConversationChanged OnConversationChangedDelegate;
 	FOnInworldPlayerConversationChangedNative& OnConversationChanged() { return OnConversationChangedDelegateNative; }
 
-	UFUNCTION()
 	bool HasAudioSession() const { return bHasAudioSession; }
+	EInworldMicrophoneMode GetMicMode() const { return MicMode; }
 
 private:
 	void UpdateConversation();
@@ -112,6 +112,7 @@ class INWORLDAIINTEGRATION_API UInworldPlayer : public UObject
 	FOnInworldPlayerConversationChangedNative OnConversationChangedDelegateNative;
 
 	bool bHasAudioSession = false;
+	EInworldMicrophoneMode MicMode = EInworldMicrophoneMode::UNKNOWN;
 };
 
 UINTERFACE(MinimalAPI, BlueprintType)

InworldAI/Source/InworldAIIntegration/Public/InworldPlayerAudioCaptureComponent.h

@@ -80,6 +80,12 @@ class INWORLDAIINTEGRATION_API UInworldPlayerAudioCaptureComponent : public UAct
     UFUNCTION(Server, Reliable, Category = "Audio")
     void ServerSetMuted(bool bInMuted);
 
+    UFUNCTION(BlueprintCallable, Category = "Audio")
+    void SetMicMode(EInworldMicrophoneMode InMicMode) { ServerSetMicMode(MicMode); }
+
+    UFUNCTION(Server, Reliable, Category = "Audio")
+    void ServerSetMicMode(EInworldMicrophoneMode InMicMode);
+
     UFUNCTION(BlueprintCallable, Category = "Devices")
     void SetCaptureDeviceById(const FString& DeviceId);
 
@@ -100,6 +106,10 @@ class INWORLDAIINTEGRATION_API UInworldPlayerAudioCaptureComponent : public UAct
     UPROPERTY(EditDefaultsOnly, BlueprintReadOnly, Category = "Audio")
     bool bMuted = false;
 
+    UPROPERTY(EditDefaultsOnly, BlueprintReadOnly, Category = "Audio")
+    EInworldMicrophoneMode MicMode = EInworldMicrophoneMode::OPEN_MIC;
+    bool bIsMicModeDirty = false;
+
 private:
 	UFUNCTION()
 	void Rep_ServerCapturingVoice();

InworldAI/Source/InworldAIIntegration/Public/InworldSession.h

@@ -124,9 +124,9 @@ class INWORLDAIINTEGRATION_API UInworldSession : public UObject
 	void SendSoundMessageToConversation(UInworldPlayer* Player, const TArray<uint8>& InputData, const TArray<uint8>& OutputData);
 
 	UFUNCTION(BlueprintCallable, Category = "Message|Audio")
-	void SendAudioSessionStart(UInworldCharacter* Character);
+	void SendAudioSessionStart(UInworldCharacter* Character, EInworldMicrophoneMode MicrophoneMode = EInworldMicrophoneMode::OPEN_MIC);
 	UFUNCTION(BlueprintCallable, Category = "Message|Audio")
-	void SendAudioSessionStartToConversation(UInworldPlayer* Player);
+	void SendAudioSessionStartToConversation(UInworldPlayer* Player, EInworldMicrophoneMode MicrophoneMode = EInworldMicrophoneMode::OPEN_MIC);
 
 	UFUNCTION(BlueprintCallable, Category = "Message|Audio")
 	void SendAudioSessionStop(UInworldCharacter* Character);