diff --git a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/AIBaseManager.cpp b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/AIBaseManager.cpp index c1d3dcb..15a7d3a 100644 --- a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/AIBaseManager.cpp +++ b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/AIBaseManager.cpp @@ -115,6 +115,17 @@ void UAIBaseManager::DeinitAIManager() BroadcastAILog(FString::Printf(TEXT("AI Manager deinitialized!")), false); } +void UAIBaseManager::OnAIReady() +{ + UAIBaseManager::SetNewState(EAvatarCoreAIState::Ready); + if (!ResponseQueue.IsEmpty()) + { + FString QueuedPrompt; + ResponseQueue.Dequeue(QueuedPrompt); + UAIBaseManager::SendResponse(QueuedPrompt, false, true); + } +} + void UAIBaseManager::SetNewState(EAvatarCoreAIState NewState, bool ForceState) { @@ -126,6 +137,13 @@ void UAIBaseManager::SetNewState(EAvatarCoreAIState NewState, bool ForceState) void UAIBaseManager::SendResponse(const FString& Response, bool NotifyDelay = false, bool TriggerResponse = true) { + if (CurrentAIState != EAvatarCoreAIState::Ready) { + ResponseQueue.Enqueue(Response); + if(CurrentAIState == EAvatarCoreAIState::Disconnected) + ActivateAI(); + return; + } + AnswerCache.Empty(); ResponseID++; LastRequest = Response; @@ -134,10 +152,11 @@ void UAIBaseManager::SendResponse(const FString& Response, bool NotifyDelay = fa ActivateAI(); return; } + UAIBaseManager::SetNewState(EAvatarCoreAIState::Processing); BroadcastAILog(FString::Printf(TEXT("AI Manager sent question/response: %s"), *Response)); if (NotifyDelay) UAIBaseManager::StartDelayedAnswerTimer(); - //Override in Child + SendResponseChild(Response, NotifyDelay, TriggerResponse); } void UAIBaseManager::RepeatText(FString TextToRepeat, bool DoRephrase) @@ -154,6 +173,8 @@ void UAIBaseManager::RepeatText(FString TextToRepeat, bool DoRephrase) void UAIBaseManager::ClearAI() { + ResponseQueue.Empty(); + if (CurrentAIState <= EAvatarCoreAIState::Ready) return; UAIBaseManager::SetNewState(EAvatarCoreAIState::Ready, true); @@ -309,6 +330,7 @@ void UAIBaseManager::ClearMCPCommand() void UAIBaseManager::CommandFinished(const FString& Command, const FString& Payload) { + UE_LOG(LogTemp, Warning, TEXT("Processing")); SetNewState(EAvatarCoreAIState::Processing); functionCallRunning = false; if (bDebugMode) diff --git a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/RealtimeAPI/AvatarCoreAIRealtime.cpp b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/RealtimeAPI/AvatarCoreAIRealtime.cpp index c960702..b276989 100644 --- a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/RealtimeAPI/AvatarCoreAIRealtime.cpp +++ b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/RealtimeAPI/AvatarCoreAIRealtime.cpp @@ -28,6 +28,7 @@ void UAvatarCoreAIRealtime::DeinitAIManager() void UAvatarCoreAIRealtime::ActivateAI() { + UAIBaseManager::SetNewState(EAvatarCoreAIState::Initializing); UAvatarCoreAIRealtime::ConnectToWebSocket(); } @@ -37,9 +38,8 @@ void UAvatarCoreAIRealtime::DeactivateAI() UAIBaseManager::DeactivateAI(); } -void UAvatarCoreAIRealtime::SendResponse(const FString& Response, bool NotifyDelay = false, bool TriggerResponse = true) +void UAvatarCoreAIRealtime::SendResponseChild(const FString& Response, bool NotifyDelay = false, bool TriggerResponse = true) { - UAIBaseManager::SendResponse(Response, NotifyDelay, TriggerResponse); UAvatarCoreAIRealtime::CreateConversationItem(Response, EOpenAIRoleType::User, TriggerResponse); } @@ -219,7 +219,7 @@ void UAvatarCoreAIRealtime::ConnectToWebSocket() void UAvatarCoreAIRealtime::DisconnectFromWebSocket() { bIsWebsocketConnected = false; - + bIsSessionInitialized = false; UAvatarCoreAIRealtime::SetNewState(EAvatarCoreAIState::Disconnected); // Clean up all accumulated audio data RequestAudioData.Empty(); @@ -295,7 +295,6 @@ void UAvatarCoreAIRealtime::CreateConversationItem(const FString& message, EOpen if (triggerResponse) { UAvatarCoreAIRealtime::CreateReseponse(); - UAvatarCoreAIRealtime::SetNewState(EAvatarCoreAIState::Processing); } } @@ -361,9 +360,11 @@ void UAvatarCoreAIRealtime::OnWebSocketConnectionError(const FString& Error) void UAvatarCoreAIRealtime::OnWebSocketConnectionConnected() { - BroadcastAILog("OpenAI Websocket connected"); - bIsWebsocketConnected = true; - UAvatarCoreAIRealtime::UpdateSession(); + if(!bIsWebsocketConnected) + { + BroadcastAILog("OpenAI Websocket connected"); + bIsWebsocketConnected = true; + } } void UAvatarCoreAIRealtime::OnWebSocketConnectionStringReceived(const FString& Message) @@ -423,10 +424,16 @@ void UAvatarCoreAIRealtime::OnWebSocketConnectionStringReceived(const FString& M } } } - else if (TypeString == "session.updated") { - UAvatarCoreAIRealtime::SetNewState(EAvatarCoreAIState::Ready); + else if (TypeString == "session.created") + { + UAvatarCoreAIRealtime::UpdateSession(); + return; + } + else if (TypeString == "session.updated") + { + UAIBaseManager::OnAIReady(); + return; } - // Match the request type if (TypeParts[0] == "session") RequestType = EOpenAIRequestType::session; diff --git a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/AIBaseManager.h b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/AIBaseManager.h index 2bcba76..83652b0 100644 --- a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/AIBaseManager.h +++ b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/AIBaseManager.h @@ -81,6 +81,10 @@ public: UFUNCTION(BlueprintCallable, Category = "AvatarCoreAI") virtual void UpdateSession() {}; + // They AI is ready to answer + UFUNCTION() + void OnAIReady(); + /** * Deinitalize AI - when going to Idle for example */ @@ -97,7 +101,10 @@ public: * Send Response/Question to the AI Model. If NotifyDelay is true call the DelayedAnswer Event when time defined in AIConfig has passed. */ UFUNCTION(BlueprintCallable, Category = "AvatarCoreAI") - virtual void SendResponse(const FString& Response, bool NotifyDelay, bool TriggerResponse); + void SendResponse(const FString& Response, bool NotifyDelay, bool TriggerResponse); + + UFUNCTION(BlueprintCallable, Category = "AvatarCoreAI") + virtual void SendResponseChild(const FString& Response, bool NotifyDelay, bool TriggerResponse) {}; /** * Make the AI Model repeat the Text. @@ -264,6 +271,8 @@ protected: //Handle Delayed Answer FTimerHandle DelayedAnswerTimer; -protected: +private: + + TQueue ResponseQueue; }; \ No newline at end of file diff --git a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/RealtimeAPI/AvatarCoreAIRealtime.h b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/RealtimeAPI/AvatarCoreAIRealtime.h index c3946ae..cbc4df2 100644 --- a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/RealtimeAPI/AvatarCoreAIRealtime.h +++ b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/RealtimeAPI/AvatarCoreAIRealtime.h @@ -156,7 +156,7 @@ public: void ActivateAI() override; void DeactivateAI() override; void UpdateSession() override; - void SendResponse(const FString& Response, bool NotifyDelay, bool TriggerResponse) override; + void SendResponseChild(const FString& Response, bool NotifyDelay, bool TriggerResponse) override; void ClearAI() override; void ConnectToWebSocket(); @@ -203,6 +203,10 @@ private: bool ResponseAudioDone = false; bool ResponseTextDone = false; + bool bIsSessionInitialized = false; + bool bWaitingForSessionUpdatedAck = false; + FString PendingSessionUpdateEventId; + // Map to store accumulated audio data for each request TMap> RequestAudioData; diff --git a/Unreal/Plugins/AvatarCore_Manager/Content/StateManagement/BFL_ProjectHelper.uasset b/Unreal/Plugins/AvatarCore_Manager/Content/StateManagement/BFL_ProjectHelper.uasset index 701de8e..323eb67 100644 --- a/Unreal/Plugins/AvatarCore_Manager/Content/StateManagement/BFL_ProjectHelper.uasset +++ b/Unreal/Plugins/AvatarCore_Manager/Content/StateManagement/BFL_ProjectHelper.uasset @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:f9d01528659c319f4e999f9ca2feb1bff11e1118a2e774379cb32f7bb7e0c0a5 -size 72498 +oid sha256:c86b0e06d695bf2a42554520c540712493ae2f4bcb20bd99825b46ac38e9787a +size 24641 diff --git a/Unreal/Plugins/AvatarCore_Manager/Content/StateManagement/BP_StateManager.uasset b/Unreal/Plugins/AvatarCore_Manager/Content/StateManagement/BP_StateManager.uasset index 89657ec..52c9962 100644 --- a/Unreal/Plugins/AvatarCore_Manager/Content/StateManagement/BP_StateManager.uasset +++ b/Unreal/Plugins/AvatarCore_Manager/Content/StateManagement/BP_StateManager.uasset @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:eeaa984d898cf627de572d29a79b81803993dd34d6cf3b0406315232410b98af -size 572469 +oid sha256:6e0ca7799eb2e02dcaa9131116a6807ada06814332c4005754d4950c078ed058 +size 600151 diff --git a/Unreal/Plugins/AvatarCore_Manager/Content/StateManagement/Commands/AICommand_TestProceedToOutro.uasset b/Unreal/Plugins/AvatarCore_Manager/Content/StateManagement/Commands/AICommand_TestProceedToOutro.uasset new file mode 100644 index 0000000..0b785e5 --- /dev/null +++ b/Unreal/Plugins/AvatarCore_Manager/Content/StateManagement/Commands/AICommand_TestProceedToOutro.uasset @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6454914d8aa5720c6833263d9a7dcfbaa8ed88ee30bfc105bbe411d3660d7737 +size 68026 diff --git a/Unreal/Plugins/AvatarCore_Manager/Content/StateManagement/CustomDialogues/BP_BaseCustomPromptDialogue.uasset b/Unreal/Plugins/AvatarCore_Manager/Content/StateManagement/CustomDialogues/BP_BaseCustomPromptDialogue.uasset new file mode 100644 index 0000000..37aeddb --- /dev/null +++ b/Unreal/Plugins/AvatarCore_Manager/Content/StateManagement/CustomDialogues/BP_BaseCustomPromptDialogue.uasset @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b431dcfe27687793a9071f87997821d284b77be9943f2751194f65d5dde02a2a +size 43108 diff --git a/Unreal/Plugins/AvatarCore_Manager/Content/StateManagement/CustomDialogues/BP_CustomDialogue_Demo.uasset b/Unreal/Plugins/AvatarCore_Manager/Content/StateManagement/CustomDialogues/BP_CustomDialogue_Demo.uasset new file mode 100644 index 0000000..b0b5a20 --- /dev/null +++ b/Unreal/Plugins/AvatarCore_Manager/Content/StateManagement/CustomDialogues/BP_CustomDialogue_Demo.uasset @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be0248e38d828043bc58b9272ae1281280d071a28b7fd8d937063d79bc06e090 +size 53756 diff --git a/Unreal/Plugins/AvatarCore_Manager/Content/StateManagement/CustomDialogues/BP_CustomDialogue_LoadFromSettings.uasset b/Unreal/Plugins/AvatarCore_Manager/Content/StateManagement/CustomDialogues/BP_CustomDialogue_LoadFromSettings.uasset new file mode 100644 index 0000000..fbc999c --- /dev/null +++ b/Unreal/Plugins/AvatarCore_Manager/Content/StateManagement/CustomDialogues/BP_CustomDialogue_LoadFromSettings.uasset @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:47f09fc85e9d8c8820948902318ddfb3371fd17b41f12d2f76a7d1d85fdc5a76 +size 63769 diff --git a/Unreal/Plugins/AvatarCore_Manager/Content/StateManagement/DataTable/PDA_Mode.uasset b/Unreal/Plugins/AvatarCore_Manager/Content/StateManagement/DataTable/PDA_Mode.uasset new file mode 100644 index 0000000..8a98124 --- /dev/null +++ b/Unreal/Plugins/AvatarCore_Manager/Content/StateManagement/DataTable/PDA_Mode.uasset @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d6993b03038ed3212001837e1ab48660f0d0cea0db61dd152f4a690b91c1c45b +size 14712 diff --git a/Unreal/Plugins/AvatarCore_Manager/Content/StateManagement/StructsAndEnums/S_ActionsToTake.uasset b/Unreal/Plugins/AvatarCore_Manager/Content/StateManagement/StructsAndEnums/S_ActionsToTake.uasset new file mode 100644 index 0000000..965f67a --- /dev/null +++ b/Unreal/Plugins/AvatarCore_Manager/Content/StateManagement/StructsAndEnums/S_ActionsToTake.uasset @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eeadae12c679cf76d42c26c6db3afcabfdf5620e2ea625c7e37cb4977c0f1a58 +size 4140 diff --git a/Unreal/Plugins/AvatarCore_Manager/Content/StateManagement/StructsAndEnums/S_AvatarInstructions.uasset b/Unreal/Plugins/AvatarCore_Manager/Content/StateManagement/StructsAndEnums/S_AvatarInstructions.uasset new file mode 100644 index 0000000..689f172 --- /dev/null +++ b/Unreal/Plugins/AvatarCore_Manager/Content/StateManagement/StructsAndEnums/S_AvatarInstructions.uasset @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1c6885a2b0db3794ad123ab2db754c1b3e761e20afb4e3338c740a7176566e5d +size 6208 diff --git a/Unreal/Plugins/AvatarCore_Manager/Content/StateManagement/StructsAndEnums/S_ProjectInstructions.uasset b/Unreal/Plugins/AvatarCore_Manager/Content/StateManagement/StructsAndEnums/S_ProjectInstructions.uasset new file mode 100644 index 0000000..b534d1b --- /dev/null +++ b/Unreal/Plugins/AvatarCore_Manager/Content/StateManagement/StructsAndEnums/S_ProjectInstructions.uasset @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:55c305dee9567d9cc17907489fcdfb981cb6f2e734f9bea0eb13459e396b77a1 +size 5383 diff --git a/Unreal/Plugins/AvatarCore_Manager/Content/StateManagement/StructsAndEnums/S_PromptDialogueType.uasset b/Unreal/Plugins/AvatarCore_Manager/Content/StateManagement/StructsAndEnums/S_PromptDialogueType.uasset new file mode 100644 index 0000000..4630b4f --- /dev/null +++ b/Unreal/Plugins/AvatarCore_Manager/Content/StateManagement/StructsAndEnums/S_PromptDialogueType.uasset @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a33b9dde4276d73ce7bc075a58ebdb2fa6cc73abed579c3eeeacc5e293f8436 +size 4033 diff --git a/Unreal/Plugins/AvatarCore_Manager/Content/StateManagement/StructsAndEnums/S_Prompt_Dialogue.uasset b/Unreal/Plugins/AvatarCore_Manager/Content/StateManagement/StructsAndEnums/S_Prompt_Dialogue.uasset new file mode 100644 index 0000000..d4e4df6 --- /dev/null +++ b/Unreal/Plugins/AvatarCore_Manager/Content/StateManagement/StructsAndEnums/S_Prompt_Dialogue.uasset @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d8c7e30691dbdf98b4e7cc960433179f5fcd3a820c61771bacac3c0ef28f86e0 +size 7669 diff --git a/Unreal/Plugins/AvatarCore_Manager/Content/StateManagement/StructsAndEnums/S_Prompt_DialogueArray.uasset b/Unreal/Plugins/AvatarCore_Manager/Content/StateManagement/StructsAndEnums/S_Prompt_DialogueArray.uasset new file mode 100644 index 0000000..24fa510 --- /dev/null +++ b/Unreal/Plugins/AvatarCore_Manager/Content/StateManagement/StructsAndEnums/S_Prompt_DialogueArray.uasset @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5008337d4fe4263b23f5b4dd74fc6f3b213674e1245073093b3f6eed6fe6e2b5 +size 4258 diff --git a/Unreal/Plugins/AvatarCore_Manager/Content/StateManagement/StructsAndEnums/S_StateProcedure.uasset b/Unreal/Plugins/AvatarCore_Manager/Content/StateManagement/StructsAndEnums/S_StateProcedure.uasset new file mode 100644 index 0000000..ec56137 --- /dev/null +++ b/Unreal/Plugins/AvatarCore_Manager/Content/StateManagement/StructsAndEnums/S_StateProcedure.uasset @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:da1a4fffcc3fd8d3c3d64637d5b26a7f5e55031faa0d9ab4775268dbe68a6a02 +size 18326 diff --git a/Unreal/Plugins/AvatarCore_Manager/Content/StateManagement/Widgets/W_BaseStateWidget.uasset b/Unreal/Plugins/AvatarCore_Manager/Content/StateManagement/Widgets/W_BaseStateWidget.uasset new file mode 100644 index 0000000..489e212 --- /dev/null +++ b/Unreal/Plugins/AvatarCore_Manager/Content/StateManagement/Widgets/W_BaseStateWidget.uasset @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0d36293b285c17b10fa3a83f847e81f9ccd426082e33576db7da49bd24b489b3 +size 177450 diff --git a/Unreal/Plugins/AvatarCore_Manager/Content/Widgets/Debug/Pages/W_DebugProjectStates.uasset b/Unreal/Plugins/AvatarCore_Manager/Content/Widgets/Debug/Pages/W_DebugProjectStates.uasset index be05457..938c226 100644 --- a/Unreal/Plugins/AvatarCore_Manager/Content/Widgets/Debug/Pages/W_DebugProjectStates.uasset +++ b/Unreal/Plugins/AvatarCore_Manager/Content/Widgets/Debug/Pages/W_DebugProjectStates.uasset @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:3e18ba822b91fb74d178d7f9b3cff961d5bd07ef5b1ad60e081605c57f403506 -size 213840 +oid sha256:d9f4e057db2b8b1d6437275ad4efe32e1073126ae5271e061355add41d496dde +size 210009