From fb4503ea8eabbb45f8f3ab45934ac036b8737edc Mon Sep 17 00:00:00 2001 From: Tillman Staffen Date: Wed, 13 May 2026 16:14:32 +0200 Subject: [PATCH] Matched with Base --- Unreal/Config/DefaultGame.ini | 1 + .../BP_AnimationTesting_Manager.uasset | 4 +- .../Data/E_AnimationTesting_Avatars.uasset | 4 +- .../AnimationTesting/M_Animation_Testing.umap | 2 +- .../Materials/MI_ProcGrid_AnimMap.uasset | 3 + .../Project/BP/Avatars/Avatar_Ben_BREX.uasset | 4 +- .../Project/BP/BP_Project_Manager.uasset | 4 +- .../EnumsAndStructs/S_ConfigSettings.uasset | 4 +- Unreal/Content/Project/Maps/M_Startup.umap | 4 +- .../2D_Environment/MM_2D_Environment.uasset | 4 +- .../AvatarCore_AI/AvatarCore_AI.Build.cs | 33 +- .../AvatarCore_AI/Private/AIBaseManager.cpp | 297 +++++---- .../AvatarCore_AI/Private/AvatarCore_AI.cpp | 55 ++ .../Private/MCP/MCPUnrealCommand.cpp | 37 +- .../OpenRouter/AvatarCoreAIOpenRouter.cpp | 570 ++++++++++++++++++ .../RealtimeAPI/AvatarCoreAIRealtime.cpp | 96 +-- .../AvatarCore_AI/Public/AIBaseConfig.h | 77 ++- .../AvatarCore_AI/Public/AIBaseManager.h | 69 ++- .../Public/AvatarCoreAIEnumsAndStructs.h | 9 + .../AvatarCore_AI/Public/AvatarCore_AI.h | 5 + .../Public/MCP/FastMCP/FastMCPConfig.h | 2 +- .../Public/MCP/MCPUnrealCommand.h | 25 +- .../OpenRouter/AvatarCoreAIOpenRouter.h | 74 +++ .../Public/OpenRouter/OpenRouterConfig.cpp | 7 + .../Public/OpenRouter/OpenRouterConfig.h | 47 ++ .../Public/RealtimeAPI/AvatarCoreAIRealtime.h | 5 +- .../Public/RealtimeAPI/RealtimeAPIConfig.h | 43 +- .../ThirdParty/MCPServer/FastMCP/.gitignore | 1 - .../FastMCP/AddDocumentsToDatabase.bat | 5 - .../FastMCP/AddDocumentsToDatabase.py | 68 --- .../MCPServer/FastMCP/FastMCPServer.bat | 142 ----- .../MCPServer/FastMCP/FastMCPServer.py | 41 -- .../MCPServer/FastMCP/StartPythonVenv.bat | 4 - .../MCPServer/FastMCP/TestSearchDatabase.bat | 5 - .../MCPServer/FastMCP/TestSearchDatabase.py | 64 -- .../MCPServer/FastMCP/WipeDatabase.bat | 9 - .../MCPServer/FastMCP/document_vectordb.py | 370 ------------ .../MCPServer/FastMCP/requirements.txt | 32 - .../MCPServer/FastMCP_ForContentFolder.zip | Bin 0 -> 11106 bytes .../Content/AvatarCoreManager.uasset | 4 +- .../States/BP_Configurable_QnA_State.uasset | 3 + .../Modules/W_AvatarCoreModuleEntry.uasset | 4 +- .../Debug/Pages/W_DebugAvatarCoreSTT.uasset | 4 +- .../Debug/Pages/W_DebugAvatarCoreTTS.uasset | 4 +- .../W_AvatarCoreStartupScreen.uasset | 4 +- .../Private/FL_AvatarCoreManager.cpp | 141 +++++ .../Public/AvatarCore_ManagerEnums.h | 64 +- .../Public/FL_AvatarCoreManager.h | 8 + ...AvatarCore_AnimInst_BodyForRetarget.uasset | 4 +- .../AnimBPs/AvatarCore_AnimInst_Face.uasset | 4 +- .../Content/BP/MetaHuman/BaseAvatar.uasset | 4 +- .../MI_GrayTexture_Body_Cascadeur.uasset | 3 + .../Materials/MI_GrayTexture_Head.uasset | 4 +- .../MI_GrayTexture_Head_Cascadeur.uasset | 3 + .../Materials/M_GrayTexture_Body.uasset | 4 +- .../Materials/M_GrayTexture_Head.uasset | 4 +- Unreal/Plugins/AvatarCore_STT/CLAUDE.md | 147 +++++ .../Preprocessor/STTPreprocessor250ms.uasset | 4 +- .../AvatarCore_STT/AvatarCore_STT.Build.cs | 8 +- .../Preprocessor/STTPreprocessorBuffer.cpp | 62 +- .../Preprocessor/STTPreprocessorConverter.cpp | 14 +- .../Preprocessor/STTPreprocessorDebugger.cpp | 4 +- .../Preprocessor/STTPreprocessorPTT.cpp | 26 +- .../Preprocessor/STTPreprocessorSpeexDSP.cpp | 12 +- .../Preprocessor/STTPreprocessorVAD.cpp | 39 +- .../Preprocessor/STTPreprocessorWebRTC.cpp | 9 +- .../Private/Processor/Azure/AzureRunnable.cpp | 11 + .../Processor/Azure/STTProcessorAzure.cpp | 151 +++-- .../Parakeet/STTParakeetProcessorBase.cpp | 103 ++-- .../RealtimeAPI/STTProcessorRealtimeAPI.cpp | 5 +- .../Processor/STTProcessorDebugSaveWav.cpp | 7 +- .../Processor/Whisper/STTProcessorWhisper.cpp | 88 +-- .../Private/Recorder/STTRecorderAudioData.cpp | 2 +- .../Private/Recorder/STTRecorderDebugFile.cpp | 8 +- .../Recorder/STTRecorderMicrophone.cpp | 2 +- .../Recorder/STTRecorderPrimaryMicrophone.cpp | 2 +- .../Recorder/STTRecorderUnrealMicrophone.cpp | 2 +- .../AvatarCore_STT/Private/STTManagerBase.cpp | 12 +- .../Public/Preprocessor/STTPreprocessorBase.h | 4 +- .../Preprocessor/STTPreprocessorBuffer.h | 17 +- .../Preprocessor/STTPreprocessorConverter.h | 2 +- .../Preprocessor/STTPreprocessorDebugger.h | 2 +- .../Public/Preprocessor/STTPreprocessorPTT.h | 7 +- .../Preprocessor/STTPreprocessorSpeexDSP.h | 2 +- .../Public/Preprocessor/STTPreprocessorVAD.h | 3 +- .../Preprocessor/STTPreprocessorWebRTC.h | 3 +- .../Processor/Azure/STTAzureProcessorConfig.h | 23 +- .../Processor/Azure/STTProcessorAzure.h | 12 +- .../Parakeet/STTParakeetProcessorBase.h | 7 +- .../Parakeet/STTParakeetProcessorConfig.h | 3 + .../RealtimeAPI/STTProcessorRealtimeAPI.h | 2 +- .../Public/Processor/STTBaseProcessorConfig.h | 7 + .../Public/Processor/STTProcessorBase.h | 2 +- .../Processor/STTProcessorDebugSaveWav.h | 2 +- .../Processor/Whisper/STTProcessorWhisper.h | 7 +- .../Public/Recorder/STTRecorderBase.h | 2 +- .../AvatarCore_STT/Public/STTManagerBase.h | 7 +- .../Source/AvatarCore_STT/Public/STTStructs.h | 62 +- .../Private/Cartesia/CartesiaTTSManager.cpp | 70 ++- .../Private/Cartesia/TTSCartesiaConfig.cpp | 4 +- .../Elevenlabs/ElevenlabsTTSConfig.cpp | 2 +- .../Elevenlabs/ElevenlabsTTSManager.cpp | 32 +- .../RealtimeAPI/TTSRealtimeAPIConfig.cpp | 2 +- .../AvatarCore_TTS/Private/TTSManagerBase.cpp | 103 ++-- .../Public/Cartesia/TTSCartesiaConfig.h | 27 +- .../Public/Elevenlabs/ElevenlabsTTSConfig.h | 27 +- .../AvatarCore_TTS/Public/TTSBaseConfig.h | 95 ++- .../AvatarCore_TTS/Public/TTSManagerBase.h | 4 + .../BSettings/Private/BSettingsSystem.cpp | 299 +++++++-- .../Source/BSettings/Public/BSettingsSystem.h | 52 +- .../Source/BTools/Private/BToolsBPLibrary.cpp | 125 ++++ .../Source/BTools/Public/BToolsBPLibrary.h | 26 + Unreal/SyncAvatarCore.bat | 53 +- 113 files changed, 2632 insertions(+), 1675 deletions(-) create mode 100644 Unreal/Content/Project/AnimationTesting/Materials/MI_ProcGrid_AnimMap.uasset create mode 100644 Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/OpenRouter/AvatarCoreAIOpenRouter.cpp create mode 100644 Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/OpenRouter/AvatarCoreAIOpenRouter.h create mode 100644 Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/OpenRouter/OpenRouterConfig.cpp create mode 100644 Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/OpenRouter/OpenRouterConfig.h delete mode 100644 Unreal/Plugins/AvatarCore_AI/Source/ThirdParty/MCPServer/FastMCP/.gitignore delete mode 100644 Unreal/Plugins/AvatarCore_AI/Source/ThirdParty/MCPServer/FastMCP/AddDocumentsToDatabase.bat delete mode 100644 Unreal/Plugins/AvatarCore_AI/Source/ThirdParty/MCPServer/FastMCP/AddDocumentsToDatabase.py delete mode 100644 Unreal/Plugins/AvatarCore_AI/Source/ThirdParty/MCPServer/FastMCP/FastMCPServer.bat delete mode 100644 Unreal/Plugins/AvatarCore_AI/Source/ThirdParty/MCPServer/FastMCP/FastMCPServer.py delete mode 100644 Unreal/Plugins/AvatarCore_AI/Source/ThirdParty/MCPServer/FastMCP/StartPythonVenv.bat delete mode 100644 Unreal/Plugins/AvatarCore_AI/Source/ThirdParty/MCPServer/FastMCP/TestSearchDatabase.bat delete mode 100644 Unreal/Plugins/AvatarCore_AI/Source/ThirdParty/MCPServer/FastMCP/TestSearchDatabase.py delete mode 100644 Unreal/Plugins/AvatarCore_AI/Source/ThirdParty/MCPServer/FastMCP/WipeDatabase.bat delete mode 100644 Unreal/Plugins/AvatarCore_AI/Source/ThirdParty/MCPServer/FastMCP/document_vectordb.py delete mode 100644 Unreal/Plugins/AvatarCore_AI/Source/ThirdParty/MCPServer/FastMCP/requirements.txt create mode 100644 Unreal/Plugins/AvatarCore_AI/Source/ThirdParty/MCPServer/FastMCP_ForContentFolder.zip create mode 100644 Unreal/Plugins/AvatarCore_Manager/Content/StateManagement/States/BP_Configurable_QnA_State.uasset create mode 100644 Unreal/Plugins/AvatarCore_MetaHuman/Content/Materials/MI_GrayTexture_Body_Cascadeur.uasset create mode 100644 Unreal/Plugins/AvatarCore_MetaHuman/Content/Materials/MI_GrayTexture_Head_Cascadeur.uasset create mode 100644 Unreal/Plugins/AvatarCore_STT/CLAUDE.md diff --git a/Unreal/Config/DefaultGame.ini b/Unreal/Config/DefaultGame.ini index 3cf4b05..8bb2a13 100644 --- a/Unreal/Config/DefaultGame.ini +++ b/Unreal/Config/DefaultGame.ini @@ -118,6 +118,7 @@ bSkipMovies=False +DirectoriesToAlwaysStageAsUFS=(Path="Schema") +DirectoriesToAlwaysStageAsNonUFS=(Path="Schema") +DirectoriesToAlwaysStageAsNonUFS=(Path="Certificates") ++DirectoriesToAlwaysStageAsNonUFS=(Path="DB") bRetainStagedDirectory=False CustomStageCopyHandler= diff --git a/Unreal/Content/Project/AnimationTesting/BP_AnimationTesting_Manager.uasset b/Unreal/Content/Project/AnimationTesting/BP_AnimationTesting_Manager.uasset index b8191df..0843f52 100644 --- a/Unreal/Content/Project/AnimationTesting/BP_AnimationTesting_Manager.uasset +++ b/Unreal/Content/Project/AnimationTesting/BP_AnimationTesting_Manager.uasset @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:46709a7ae020f20fa8ce76e5712f4d1a832e8b3594c8e8093fe75c1bb9d789c9 -size 643441 +oid sha256:dbf1b9dbc5d89cb90030fd39f551d441cf82d073391640e4e32c3308b36365a1 +size 382167 diff --git a/Unreal/Content/Project/AnimationTesting/Data/E_AnimationTesting_Avatars.uasset b/Unreal/Content/Project/AnimationTesting/Data/E_AnimationTesting_Avatars.uasset index 326bfd1..1ddec8d 100644 --- a/Unreal/Content/Project/AnimationTesting/Data/E_AnimationTesting_Avatars.uasset +++ b/Unreal/Content/Project/AnimationTesting/Data/E_AnimationTesting_Avatars.uasset @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:74a4b7c56f65a024fcd0c9aceb88612414854adb71e2abf7294c7f349bea8a44 -size 1900 +oid sha256:3d6d42d63c84b9188c51602fdfbc8daec85be3938048154fb724b96b49317162 +size 2722 diff --git a/Unreal/Content/Project/AnimationTesting/M_Animation_Testing.umap b/Unreal/Content/Project/AnimationTesting/M_Animation_Testing.umap index e7ab3d7..9da99f2 100644 --- a/Unreal/Content/Project/AnimationTesting/M_Animation_Testing.umap +++ b/Unreal/Content/Project/AnimationTesting/M_Animation_Testing.umap @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:35267261934a5a82b81638bcfc8efff1353b34feb5486c6bafbf80f45f92e65e +oid sha256:7ae90ef8e72e1c65babf293c86079313b1120e43af9d1d45196705655464fcba size 155359 diff --git a/Unreal/Content/Project/AnimationTesting/Materials/MI_ProcGrid_AnimMap.uasset b/Unreal/Content/Project/AnimationTesting/Materials/MI_ProcGrid_AnimMap.uasset new file mode 100644 index 0000000..af1681d --- /dev/null +++ b/Unreal/Content/Project/AnimationTesting/Materials/MI_ProcGrid_AnimMap.uasset @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bce6f803482d3a7132b655d70f9a12840ea7302e1b2ea434dba51f9109990d23 +size 11525 diff --git a/Unreal/Content/Project/BP/Avatars/Avatar_Ben_BREX.uasset b/Unreal/Content/Project/BP/Avatars/Avatar_Ben_BREX.uasset index cc334a3..cc2486c 100644 --- a/Unreal/Content/Project/BP/Avatars/Avatar_Ben_BREX.uasset +++ b/Unreal/Content/Project/BP/Avatars/Avatar_Ben_BREX.uasset @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:13f3f3ff65addaca7860de24662df04ab097dd38ab5543dc9125b1555189e57c -size 58142 +oid sha256:ceea9e3126afaaeab683a17d99a79ea88c453aa43571c2c4e6bd2106060009ce +size 58086 diff --git a/Unreal/Content/Project/BP/BP_Project_Manager.uasset b/Unreal/Content/Project/BP/BP_Project_Manager.uasset index bc29230..ca8f581 100644 --- a/Unreal/Content/Project/BP/BP_Project_Manager.uasset +++ b/Unreal/Content/Project/BP/BP_Project_Manager.uasset @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:33665c78ce97f4d9dc7d5cca981b2f1c34bfe4925e76126b43351b122f75a272 -size 2326491 +oid sha256:cbd4979d1be26882f0eb2dc7a97209e9ef5260f371383f9c126ff38256d520ae +size 2481470 diff --git a/Unreal/Content/Project/BP/EnumsAndStructs/S_ConfigSettings.uasset b/Unreal/Content/Project/BP/EnumsAndStructs/S_ConfigSettings.uasset index 9457f8e..b8a3ed6 100644 --- a/Unreal/Content/Project/BP/EnumsAndStructs/S_ConfigSettings.uasset +++ b/Unreal/Content/Project/BP/EnumsAndStructs/S_ConfigSettings.uasset @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:7ec73128a21246128018cf07feeeaf36fbb91e6175da6b819cb72970d0be7817 -size 59922 +oid sha256:5a0e3faa1466076c0d45f1fd97676f3b7a3b402c4d68430dacf08594d79bc81d +size 69366 diff --git a/Unreal/Content/Project/Maps/M_Startup.umap b/Unreal/Content/Project/Maps/M_Startup.umap index aeeb543..4da472d 100644 --- a/Unreal/Content/Project/Maps/M_Startup.umap +++ b/Unreal/Content/Project/Maps/M_Startup.umap @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:bd07909883d02c91c3c587aac8f0d0f1f19d7ca97c3787c3be4207f2c4e29377 -size 168834 +oid sha256:27527e1ec7a56ba04281237f9cb355bcc889abfed0158967f01db26d5d5ef7d5 +size 159627 diff --git a/Unreal/Content/Project/Materials/2D_Environment/MM_2D_Environment.uasset b/Unreal/Content/Project/Materials/2D_Environment/MM_2D_Environment.uasset index e2d25db..fabc150 100644 --- a/Unreal/Content/Project/Materials/2D_Environment/MM_2D_Environment.uasset +++ b/Unreal/Content/Project/Materials/2D_Environment/MM_2D_Environment.uasset @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:6bea808945faa5e21438123b6eb7e666584b73ac6a2e6b844b55b16cee8c51c1 -size 31234 +oid sha256:e5394e513d67cbae8ef01c45a95b0ec0336723a692f28026ea62c5665b76c81b +size 31346 diff --git a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/AvatarCore_AI.Build.cs b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/AvatarCore_AI.Build.cs index b833ed7..8968b14 100644 --- a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/AvatarCore_AI.Build.cs +++ b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/AvatarCore_AI.Build.cs @@ -55,36 +55,17 @@ public class AvatarCore_AI : ModuleRules // ... add private dependencies that you statically link with here ... } ); - - - DynamicallyLoadedModuleNames.AddRange( + + // Will only be added in Editor Build + if(Target.bBuildEditor) + PrivateDependencyModuleNames.Add("DeveloperToolSettings"); + + + DynamicallyLoadedModuleNames.AddRange( new string[] { // ... add any modules that your module loads dynamically here ... } ); - - // Package MCPServer folder with shipping builds - string MCPServerPath = Path.Combine(ModuleDirectory, "..", "ThirdParty", "MCPServer"); - if (Directory.Exists(MCPServerPath)) - { - // Add all files in the MCPServer directory recursively - foreach (string FilePath in Directory.GetFiles(MCPServerPath, "*", SearchOption.AllDirectories)) - { - string RelativePath = Path.GetRelativePath(MCPServerPath, FilePath); - string TargetPath = "$(BinaryOutputDir)/MCPServer/" + RelativePath.Replace('\\', '/'); - RuntimeDependencies.Add(TargetPath, FilePath, StagedFileType.NonUFS); - } - PublicDefinitions.Add("WITH_MCP_SERVER=1"); - } - else - { - PublicDefinitions.Add("WITH_MCP_SERVER=0"); - } - - // Ensure ThirdParty/PiperTTS is packaged in all builds (including shipping) - string MCPPath = System.IO.Path.Combine(ModuleDirectory, "..", "ThirdParty", "MCPServer"); - RuntimeDependencies.Add(System.IO.Path.Combine(MCPPath, "*.*")); - RuntimeDependencies.Add(System.IO.Path.Combine(MCPPath, "**", "*.*")); } } diff --git a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/AIBaseManager.cpp b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/AIBaseManager.cpp index 4786aa8..fcc076a 100644 --- a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/AIBaseManager.cpp +++ b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/AIBaseManager.cpp @@ -27,31 +27,24 @@ void UAIBaseManager::InitAIManager(UAIBaseConfig* AIConfig, bool DebugMode, AAct World = WorldReferenceActor->GetWorld(); } - // Add all UnrealCommands to root + // Register command classes; instances are created on-demand when a command is invoked for (TSubclassOf CommandClass : CurrentConfig->UnrealCommands) { - if (CommandClass != nullptr) + if (!CommandClass) continue; + UMCPUnrealCommand* CDO = CommandClass->GetDefaultObject(); + if (CDO->GetCommandName().IsEmpty()) { - UMCPUnrealCommand* Command = NewObject(this, CommandClass); - if (Command->GetCommandName().IsEmpty()) - { - BroadcastAIError(TEXT("Command has empty CommandName!"), EAvatarCoreAIError::InvalidConfig); - continue; - } - else - { - Command->AddToRoot(); - Command->InitMCPCommand(World); - UnrealCommands.Add(Command); - UnrealCommandsToolInfos.Add(Command->GetToolInfo()); - } + BroadcastAIError(TEXT("Command has empty CommandName!"), EAvatarCoreAIError::InvalidConfig); + continue; } + UnrealCommandClasses.Add(CommandClass); + UnrealCommandsToolInfos.Add(CDO->GetToolInfo()); } this->AddToRoot(); bIsRooted = true; - if(AIConfig->bUseMCPServer && AIConfig->MCPManagerClass != nullptr) + if(AIConfig->GlobalAISettings.bUseMCPServer && AIConfig->MCPManagerClass != nullptr) { BroadcastAILog(TEXT("Initializing MCP Server..."), true); @@ -61,8 +54,8 @@ void UAIBaseManager::InitAIManager(UAIBaseConfig* AIConfig, bool DebugMode, AAct // Bind to MCP events for forwarding MCPManager->OnMCPLog.AddDynamic(this, &UAIBaseManager::OnMCPLogReceived); - MCPManager->OnMCPCommandDone.AddDynamic(this, &UAIBaseManager::CommandFinished); - MCPManager->OnMCPCommandFailed.AddDynamic(this, &UAIBaseManager::CommandFailed); + MCPManager->OnMCPCommandDone.AddDynamic(this, &UAIBaseManager::MCPCommandFinished); + MCPManager->OnMCPCommandFailed.AddDynamic(this, &UAIBaseManager::MCPCommandFailed); MCPManager->OnMCPManagerError.AddDynamic(this, &UAIBaseManager::OnMCPErrorReceived); MCPManager->OnMCPManagerStateChanged.AddDynamic(this, &UAIBaseManager::OnMCPStateChanged); @@ -102,15 +95,11 @@ void UAIBaseManager::DeinitAIManager() CurrentConfig->MCPConfig->RemoveFromRoot(); } + ActiveCommands.Empty(); + UnrealCommandClasses.Empty(); + UnrealCommandsToolInfos.Empty(); + if(CurrentConfig && CurrentConfig->IsRooted()){ - // Remove all UnrealCommands from root before clearing config - for (UMCPUnrealCommand* Command : UnrealCommands) - { - if (Command && Command->IsRooted()) - { - Command->RemoveFromRoot(); - } - } CurrentConfig->RemoveFromRoot(); CurrentConfig = nullptr; } @@ -130,7 +119,7 @@ void UAIBaseManager::OnAIReady() { FAIMessage QueuedPrompt; ResponseQueue.Dequeue(QueuedPrompt); - UAIBaseManager::SendResponse(QueuedPrompt, false, true); + UAIBaseManager::SendResponse(QueuedPrompt, false); } } @@ -154,20 +143,16 @@ void UAIBaseManager::SetNewState(EAvatarCoreAIState NewState, bool ForceState) } } -void UAIBaseManager::SendResponse(FAIMessage Message, bool NotifyDelay, bool TriggerResponse) +void UAIBaseManager::SendResponse(FAIMessage Message, bool NotifyDelay) { - if (CurrentAIState != EAvatarCoreAIState::Ready && TriggerResponse) { - FAIMessage tmpPrompt; - tmpPrompt.Message = Message.Message; - tmpPrompt.Role = Message.Role; - ResponseQueue.Enqueue(tmpPrompt); + if (CurrentAIState != EAvatarCoreAIState::Ready) + { + ResponseQueue.Enqueue(Message); if(CurrentAIState == EAvatarCoreAIState::Disconnected) - ActivateAI(); + ActivateAI(); return; } - AddMessageToArray(Message); - AnswerCache.Empty(); ResponseID++; LastRequest = Message.Message; @@ -176,11 +161,14 @@ void UAIBaseManager::SendResponse(FAIMessage Message, bool NotifyDelay, bool Tri ActivateAI(); return; } - UAIBaseManager::SetNewState(EAvatarCoreAIState::Processing); + if (Message.bTriggerResponse) + UAIBaseManager::SetNewState(EAvatarCoreAIState::Processing); BroadcastAILog(FString::Printf(TEXT("AI Manager sent question/response: %s"), *Message.Message)); if (NotifyDelay) UAIBaseManager::StartDelayedAnswerTimer(); - SendResponseChild(Message, NotifyDelay, TriggerResponse); + SendResponseChild(Message, NotifyDelay); + if(Message.Role == EAvatarCoreAIPromptRole::User || Message.Role == EAvatarCoreAIPromptRole::Tool) + AddMessageToArray(Message); } void UAIBaseManager::RepeatText(FString TextToRepeat, bool DoRephrase) @@ -195,7 +183,7 @@ void UAIBaseManager::RepeatText(FString TextToRepeat, bool DoRephrase) FAIMessage tmpPrompt; tmpPrompt.Message = Instruction; tmpPrompt.Role = EAvatarCoreAIPromptRole::System; - SendResponse(tmpPrompt, false, true); + SendResponse(tmpPrompt, false); } void UAIBaseManager::ClearAI() @@ -211,21 +199,22 @@ void UAIBaseManager::ClearAI() //Extend in Child } -void UAIBaseManager::BroadcastAILog(const FString& Message, bool ShowAlways) +void UAIBaseManager::BroadcastAILog(const FString& Message, bool ShowAlways, bool VeryVerbose) { if (!bDebugMode && !ShowAlways) return; if (IsInGameThread()) { - OnAILog.Broadcast(Message); + OnAILog.Broadcast(Message, VeryVerbose); } else { FString Copy = Message; - AsyncTask(ENamedThreads::GameThread, [this, Copy]() + bool CopyVerbosity = VeryVerbose; + AsyncTask(ENamedThreads::GameThread, [this, Copy, CopyVerbosity]() { - OnAILog.Broadcast(Copy); + OnAILog.Broadcast(Copy, CopyVerbosity); }); } } @@ -249,36 +238,27 @@ void UAIBaseManager::BroadcastAIError(const FString& ErrorMessage, EAvatarCoreAI // Thread-safe critical section for AnswerCache static FCriticalSection AnswerCacheCriticalSection; -void UAIBaseManager::AddUnrealCommand(UMCPUnrealCommand* Command) -{; - if (Command->GetCommandName().IsEmpty()) +void UAIBaseManager::AddUnrealCommand(TSubclassOf CommandClass) +{ + if (!CommandClass) return; + UMCPUnrealCommand* CDO = CommandClass->GetDefaultObject(); + if (CDO->GetCommandName().IsEmpty()) { BroadcastAIError(TEXT("Command has empty CommandName!"), EAvatarCoreAIError::InvalidConfig); return; } - else - { - UnrealCommands.Add(Command); - UnrealCommandsToolInfos.Add(Command->GetToolInfo()); - if (!Command->IsRooted()) - { - Command->AddToRoot(); - } - } + UnrealCommandClasses.Add(CommandClass); + UnrealCommandsToolInfos.Add(CDO->GetToolInfo()); } void UAIBaseManager::RemoveUnrealCommand(const FString& CommandName) { - for (int32 i = UnrealCommands.Num() - 1; i >= 0; --i) + for (int32 i = UnrealCommandClasses.Num() - 1; i >= 0; --i) { - UMCPUnrealCommand* Command = UnrealCommands[i]; - if (Command && Command->GetCommandName() == CommandName) + UMCPUnrealCommand* CDO = UnrealCommandClasses[i]->GetDefaultObject(); + if (CDO && CDO->GetCommandName().Equals(CommandName, ESearchCase::IgnoreCase)) { - if (Command->IsRooted()) - { - Command->RemoveFromRoot(); - } - UnrealCommands.RemoveAt(i); + UnrealCommandClasses.RemoveAt(i); UnrealCommandsToolInfos.RemoveAt(i); break; } @@ -300,23 +280,27 @@ TArray UAIBaseManager::GetAvailableCommands() } } -void UAIBaseManager::RunMCPCommand(FString CommandName, FString Payload) +void UAIBaseManager::RunMCPCommand(FString CommandName, FString Payload, FString ToolCallId) { ClearRequestTimeout(); - + if (!CurrentConfig) { BroadcastAIError(TEXT("No config loaded for RunCommand"), EAvatarCoreAIError::InvalidConfig); return; } - UMCPUnrealCommand* FoundUnrealCommand = nullptr; - for (UMCPUnrealCommand* Command : UnrealCommands) { - if (Command && Command->GetCommandName().ToLower().Equals(CommandName.ToLower())) { - FoundUnrealCommand = Command; + + // Find the registered class matching the command name + TSubclassOf FoundClass = nullptr; + for (TSubclassOf CmdClass : UnrealCommandClasses) + { + if (CmdClass && CmdClass->GetDefaultObject()->GetCommandName().Equals(CommandName, ESearchCase::IgnoreCase)) + { + FoundClass = CmdClass; break; } } - if (!FoundUnrealCommand && !MCPManager || !FoundUnrealCommand && !MCPManager->HasCommand(CommandName)) { + if (!FoundClass && (!MCPManager || !MCPManager->HasCommand(CommandName))) { BroadcastAIError(FString::Printf(TEXT("Command '%s' not found in Unreal Commands or MCP"), *CommandName), EAvatarCoreAIError::MCPError); return; } @@ -325,36 +309,45 @@ void UAIBaseManager::RunMCPCommand(FString CommandName, FString Payload) SetNewState(EAvatarCoreAIState::GettingInfo); functionCallRunning = true; - if (FoundUnrealCommand) + if (FoundClass) { UWorld* World = nullptr; - if (WorldReferenceActor.IsValid()) { + if (WorldReferenceActor.IsValid()) World = WorldReferenceActor->GetWorld(); - } - // Remove all previous bindings - FoundUnrealCommand->OnCommandDone.Clear(); - FoundUnrealCommand->OnCommandFailed.Clear(); - // Bind events to this instance - FoundUnrealCommand->OnCommandDone.AddDynamic(this, &UAIBaseManager::CommandFinished); - FoundUnrealCommand->OnCommandFailed.AddDynamic(this, &UAIBaseManager::CommandFailed); - // Execute the command - FoundUnrealCommand->Execute(World, Payload); + + // Create a fresh instance for this invocation; ActiveCommands (UPROPERTY) keeps it alive + UMCPUnrealCommand* Cmd = NewObject(this, FoundClass); + Cmd->Id = ToolCallId; + Cmd->SetWorldContext(WorldReferenceActor.Get()); + ActiveCommands.Add(Cmd); + + Cmd->OnCommandDone.AddDynamic(this, &UAIBaseManager::CommandFinished); + Cmd->OnCommandFailed.AddDynamic(this, &UAIBaseManager::CommandFailed); + + Cmd->InitMCPCommand(World); + Cmd->Execute(World, Payload); return; } - if(MCPManager && MCPManager->HasCommand(CommandName)) + if (MCPManager && MCPManager->HasCommand(CommandName)) { + if (!ToolCallId.IsEmpty()) + MCPToolCallIds.Add(CommandName, ToolCallId); MCPManager->ExecuteCommand(CommandName, Payload); - return; } } void UAIBaseManager::ClearMCPCommand() { - for (UMCPUnrealCommand* Command : UnrealCommands) { - Command->OnCommandDone.Clear(); - Command->OnCommandFailed.Clear(); + for (UMCPUnrealCommand* Command : ActiveCommands) + { + if (Command) + { + Command->OnCommandDone.Clear(); + Command->OnCommandFailed.Clear(); + } } + ActiveCommands.Empty(); } FString UAIBaseManager::GetRoleAsString(EAvatarCoreAIPromptRole Role) @@ -370,29 +363,76 @@ FString UAIBaseManager::GetRoleAsString(EAvatarCoreAIPromptRole Role) } } -void UAIBaseManager::CommandFinished(const FString& Command, const FString& Payload) +void UAIBaseManager::CommandFinished(const FAIMessage& Message) +{ + ActiveCommands.RemoveAll([&Message](UMCPUnrealCommand* Cmd) + { + return Cmd && (Message.Id.IsEmpty() || Cmd->Id.Equals(Message.Id)); + }); + + SetNewState(EAvatarCoreAIState::Ready); + functionCallRunning = false; + if (bDebugMode) + BroadcastAILog(FString::Printf(TEXT("Command ran successfully. Answer: %s"), *Message.Message), true); + else + BroadcastAILog(TEXT("Command ran successfully."), true); + + SendResponse(Message, false); +} + +void UAIBaseManager::CommandFailed(const FAIMessage& Message) { + ActiveCommands.RemoveAll([&Message](UMCPUnrealCommand* Cmd) + { + return Cmd && (Message.Id.IsEmpty() || Cmd->Id.Equals(Message.Id)); + }); + + functionCallRunning = false; + SetNewState(EAvatarCoreAIState::Ready); + BroadcastAILog(FString::Printf(TEXT("Command failed. Sending: %s"), *Message.Message), true); + SendResponse(Message, false); +} + +void UAIBaseManager::MCPCommandFinished(const FString& Command, const FString& Payload) +{ + FString FoundId; + FString* MCPId = MCPToolCallIds.Find(Command); + if (MCPId) + { + FoundId = *MCPId; + MCPToolCallIds.Remove(Command); + } + SetNewState(EAvatarCoreAIState::Ready); functionCallRunning = false; if (bDebugMode) - BroadcastAILog(FString::Printf(TEXT("Command '%s' ran successfully. Answer: %s"), *Command, *Payload), true); + BroadcastAILog(FString::Printf(TEXT("MCP Command '%s' ran successfully. Answer: %s"), *Command, *Payload), true); else - BroadcastAILog(FString::Printf(TEXT("Command '%s' ran successfully."), *Command), true); + BroadcastAILog(FString::Printf(TEXT("MCP Command '%s' ran successfully."), *Command), true); + FAIMessage FinishedCommandMessage; - FinishedCommandMessage.Role = EAvatarCoreAIPromptRole::System; FinishedCommandMessage.Message = Payload; - SendResponse(FinishedCommandMessage, false, true); + if (!FoundId.IsEmpty()) + { + FinishedCommandMessage.Role = EAvatarCoreAIPromptRole::Tool; + FinishedCommandMessage.Id = FoundId; + } + else + { + FinishedCommandMessage.Role = EAvatarCoreAIPromptRole::System; + } + SendResponse(FinishedCommandMessage, false); } -void UAIBaseManager::CommandFailed(const FString& Command, const FString& Payload) +void UAIBaseManager::MCPCommandFailed(const FString& Command, const FString& Payload) { functionCallRunning = false; SetNewState(EAvatarCoreAIState::Ready); - BroadcastAILog(FString::Printf(TEXT("Command '%s' failed. Sending: %s"), *Command, *Payload), true); + BroadcastAILog(FString::Printf(TEXT("MCP Command '%s' failed. Sending: %s"), *Command, *Payload), true); FAIMessage FailedCommandMessage; FailedCommandMessage.Role = EAvatarCoreAIPromptRole::System; FailedCommandMessage.Message = Payload; - SendResponse(FailedCommandMessage, false, true); + SendResponse(FailedCommandMessage, false); } void UAIBaseManager::AddMessageToArray(FAIMessage NewMessage) @@ -400,7 +440,7 @@ void UAIBaseManager::AddMessageToArray(FAIMessage NewMessage) PreviousMessages.Add(NewMessage); // Remove oldest if over limit - if (CurrentConfig->MaxMessages > -1 && PreviousMessages.Num() > CurrentConfig->MaxMessages) + if (CurrentConfig->GlobalAISettings.MaxMessages > -1 && PreviousMessages.Num() > CurrentConfig->GlobalAISettings.MaxMessages) { PreviousMessages.RemoveAt(0); // removes oldest (first element) } @@ -414,13 +454,13 @@ TArray UAIBaseManager::GetAllPreviousMessage() void UAIBaseManager::StartDelayedAnswerTimer() { UAIBaseManager::ClearDelayedAnswerTimer(); - if(CurrentConfig->DelayAnswerSeconds > 0.0f) + if(CurrentConfig->GlobalAISettings.DelayAnswerSeconds > 0.0f) { GetWorld()->GetTimerManager().SetTimer( DelayedAnswerTimer, this, &UAIBaseManager::OnDelayedAnswer, - CurrentConfig->DelayAnswerSeconds, + CurrentConfig->GlobalAISettings.DelayAnswerSeconds, false ); } @@ -473,64 +513,85 @@ void UAIBaseManager::OnAIResponse(const FString& Chunk, bool IsFinal) { FAIMessage tmpAIAnswer; tmpAIAnswer.Role = EAvatarCoreAIPromptRole::Assistant; - tmpAIAnswer.Message = Chunk; + tmpAIAnswer.Message = UpdatedAnswer; AddMessageToArray(tmpAIAnswer); } } -void UAIBaseManager::AddSystemInstructions(const TArray SystemInstructions, bool WipeCurrent = true) +void UAIBaseManager::AddSystemInstructions(const TArray SystemInstructions, bool AutoSyncWithAI = false) { - if (WipeCurrent) - ClearAllSystemInstructios(); - CurrentConfig->SystemPrompts.Append(SystemInstructions); - + for (const FSystemInstruction& Item : SystemInstructions) + { + AddSystemInstruction(Item, false, false); + } + if (AutoSyncWithAI) + UpdateSession(); } -void UAIBaseManager::AddSystemInstruction(const FName Name, const FString NewSystemInstruction, bool AddAsFirst = false) +void UAIBaseManager::AddSystemInstruction(const FSystemInstruction SystemInstruction, bool AddAsFirst = false, bool AutoSyncWithAI = false) { - UAIBaseManager::RemoveSystemInstruction(Name); - FSystemInstruction tmpEntry; - tmpEntry.Name = Name; - tmpEntry.Instruction = NewSystemInstruction; + UAIBaseManager::RemoveSystemInstruction(SystemInstruction.Name, false); if(AddAsFirst) { TArray tmpSystemPrompts; - tmpSystemPrompts.Add(tmpEntry); + tmpSystemPrompts.Add(SystemInstruction); tmpSystemPrompts.Append(CurrentConfig->SystemPrompts); CurrentConfig->SystemPrompts = tmpSystemPrompts; } else - CurrentConfig->SystemPrompts.Add(tmpEntry); + CurrentConfig->SystemPrompts.Add(SystemInstruction); - BroadcastAILog(FString::Printf(TEXT("AI Manager added System Instruction %s"), *Name.ToString())); + BroadcastAILog(FString::Printf(TEXT("AI Manager added System Instruction %s"), *SystemInstruction.Name.ToString())); + if (AutoSyncWithAI) + UpdateSession(); } -void UAIBaseManager::ClearAllSystemInstructios() +void UAIBaseManager::ClearAllSystemInstructions(bool AutoSyncWithAI = false) { CurrentConfig->SystemPrompts.Empty(); BroadcastAILog(FString::Printf(TEXT("AI Manager wiped all System Instructions"))); UAIBaseManager::AddRepeatSystemInstruction(); + if (AutoSyncWithAI) + UpdateSession(); } void UAIBaseManager::AddRepeatSystemInstruction() { - UAIBaseManager::AddSystemInstruction(TEXT("Repeat Text"), TEXT("If the text starts with [REPEAT], repeat the text exactly word for word."), true); - UAIBaseManager::AddSystemInstruction(TEXT("Rephrase Text"), TEXT("If the text starts with [REPHRASE], repeat the text in your own words without stating that you are rephrasing."), true); + FSystemInstruction repeatInstruction; + repeatInstruction.Name = "Repeat Text"; + repeatInstruction.Instruction = "If the text starts with [REPEAT], repeat the text exactly word for word."; + FSystemInstruction rephraseInstruction; + rephraseInstruction.Name = "Rephrase Text"; + rephraseInstruction.Instruction = "If the text starts with [REPHRASE], repeat the text in your own words without stating that you are rephrasing."; + UAIBaseManager::AddSystemInstruction(repeatInstruction); + UAIBaseManager::AddSystemInstruction(rephraseInstruction); } -void UAIBaseManager::RemoveSystemInstruction(const FName Name) +void UAIBaseManager::RemoveSystemInstruction(FName SystemInstruction, bool AutoSyncWithAI = false) { // Iterate in reverse to safely remove while iterating for (int32 i = CurrentConfig->SystemPrompts.Num() - 1; i >= 0; --i) { - if (CurrentConfig->SystemPrompts[i].Name == Name) + if (CurrentConfig->SystemPrompts[i].Name == SystemInstruction) { CurrentConfig->SystemPrompts.RemoveAt(i); - BroadcastAILog(FString::Printf(TEXT("AI Manager removed System Instruction %s"), *Name.ToString())); + BroadcastAILog(FString::Printf(TEXT("AI Manager removed System Instruction %s"), *SystemInstruction.ToString())); } } + if (AutoSyncWithAI) + UpdateSession(); +} + +void UAIBaseManager::RemoveSystemInstructions(const TArray SystemInstructions, bool AutoSyncWithAI) +{ + for (const FName& Item : SystemInstructions) + { + RemoveSystemInstruction(Item, false); + } + if (AutoSyncWithAI) + UpdateSession(); } FString UAIBaseManager::GetSystemInstructionPromptString(bool AsJsonString = false) @@ -588,7 +649,7 @@ void UAIBaseManager::ResetRequestTimeout() RequestTimeoutTimer, // handle to cancel timer at a later time this, // the owning object &UAIBaseManager::OnRequestTimeout, // function to call on elapsed - CurrentConfig->RequestTimeout, // float delay until elapsed slightly shorter than chunk length + CurrentConfig->GlobalAISettings.RequestTimeout, // float delay until elapsed slightly shorter than chunk length false); // looping? } diff --git a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/AvatarCore_AI.cpp b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/AvatarCore_AI.cpp index f63997f..24a8fe5 100644 --- a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/AvatarCore_AI.cpp +++ b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/AvatarCore_AI.cpp @@ -2,11 +2,19 @@ #include "AvatarCore_AI.h" +#if WITH_EDITOR +#include "Settings/ProjectPackagingSettings.h" +#endif + #define LOCTEXT_NAMESPACE "FAvatarCore_AIModule" void FAvatarCore_AIModule::StartupModule() { // This code will execute after your module is loaded into memory; the exact timing is specified in the .uplugin file per-module +#if WITH_EDITOR +// Add the Schema directory to packaging settings immediately when module starts + FAvatarCore_AIModule::AddDBDirectoryToPackaging(); +#endif } void FAvatarCore_AIModule::ShutdownModule() @@ -15,6 +23,53 @@ void FAvatarCore_AIModule::ShutdownModule() // we call this function before unloading the module. } +#if WITH_EDITOR +void FAvatarCore_AIModule::AddDBDirectoryToPackaging() +{ + // Get the project packaging settings + UProjectPackagingSettings* PackagingSettings = GetMutableDefault(); + + if (PackagingSettings) + { + // Define the DB directory path relative to Content + const FString DBDir = TEXT("DB"); + + // Check if the directory is already in the list + bool bAlreadyExists = false; + for (const FDirectoryPath& ExistingPath : PackagingSettings->DirectoriesToAlwaysStageAsNonUFS) + { + if (ExistingPath.Path == DBDir) + { + bAlreadyExists = true; + break; + } + } + + // Add the directory if it doesn't exist + if (!bAlreadyExists) + { + FDirectoryPath NewPath; + NewPath.Path = DBDir; + PackagingSettings->DirectoriesToAlwaysStageAsNonUFS.Add(NewPath); + + // Save the settings to disk + PackagingSettings->TryUpdateDefaultConfigFile(); + + UE_LOG(LogTemp, Log, TEXT("Database directory added to packaging settings")); + } + else + { + UE_LOG(LogTemp, Log, TEXT("Database directory already exists in packaging settings")); + } + } + else + { + UE_LOG(LogTemp, Warning, TEXT("Could not access Project Packaging Settings")); + } + +} +#endif + #undef LOCTEXT_NAMESPACE IMPLEMENT_MODULE(FAvatarCore_AIModule, AvatarCore_AI) \ No newline at end of file diff --git a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/MCP/MCPUnrealCommand.cpp b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/MCP/MCPUnrealCommand.cpp index b979222..4bf0b58 100644 --- a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/MCP/MCPUnrealCommand.cpp +++ b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/MCP/MCPUnrealCommand.cpp @@ -52,24 +52,27 @@ void UMCPUnrealCommand::StartTimeout() } } -void UMCPUnrealCommand::FinishCommand(const FString& Payload) +void UMCPUnrealCommand::FinishCommand(const FString& Payload, bool bTriggerResponse) { - OnCommandDone.Broadcast(GetCommandName(), Payload); - // Always clear timeout, even if already finished + FAIMessage Msg; + Msg.Role = EAvatarCoreAIPromptRole::Tool; + Msg.Message = Payload; + Msg.Id = Id; + Msg.bTriggerResponse = bTriggerResponse; + OnCommandDone.Broadcast(Msg); if (GetWorld()) - { GetWorld()->GetTimerManager().ClearTimer(TimeoutHandle); - } } -void UMCPUnrealCommand::FailCommand(const FString& Payload) +void UMCPUnrealCommand::FailCommand(const FString& Payload, bool bTriggerResponse) { - OnCommandFailed.Broadcast(GetCommandName(), Payload); - // Always clear timeout, even if already finished + FAIMessage Msg; + Msg.Role = EAvatarCoreAIPromptRole::System; + Msg.Message = Payload; + Msg.bTriggerResponse = bTriggerResponse; + OnCommandFailed.Broadcast(Msg); if (GetWorld()) - { GetWorld()->GetTimerManager().ClearTimer(TimeoutHandle); - } } void UMCPUnrealCommand::OnTimeout() @@ -89,11 +92,21 @@ AActor* UMCPUnrealCommand::GetActorOfClass(UWorld* World, TSubclassOf Ac return nullptr; } +void UMCPUnrealCommand::SetWorldContext(UObject* NewWorldContext) +{ + RequiredWorldContext = NewWorldContext; +} + +UObject* UMCPUnrealCommand::GetWorldContextObject() const +{ + return RequiredWorldContext; +} + UWorld* UMCPUnrealCommand::GetWorld() const { + if (RequiredWorldContext) + return RequiredWorldContext->GetWorld(); if (const UObject* Outer = GetOuter()) - { return Outer->GetWorld(); - } return nullptr; } diff --git a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/OpenRouter/AvatarCoreAIOpenRouter.cpp b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/OpenRouter/AvatarCoreAIOpenRouter.cpp new file mode 100644 index 0000000..e257e9d --- /dev/null +++ b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/OpenRouter/AvatarCoreAIOpenRouter.cpp @@ -0,0 +1,570 @@ +// Fill out your copyright notice in the Description page of Project Settings. + +#include "OpenRouter/AvatarCoreAIOpenRouter.h" +#include "HttpModule.h" +#include "Interfaces/IHttpRequest.h" +#include "Interfaces/IHttpResponse.h" +#include "Dom/JsonObject.h" +#include "Dom/JsonValue.h" +#include "Serialization/JsonReader.h" +#include "Serialization/JsonWriter.h" +#include "Serialization/JsonSerializer.h" + +// Appends incoming HTTP response bytes directly to a shared TArray. +// Called from the HTTP module's background thread; the lock guards against concurrent +// reads in OnResponseProgress (game thread). Both the manager and the archive hold +// shared refs to the buffer and lock, so neither is destroyed under a live callback. +class FSSEReceiveArchive final : public FArchive +{ +public: + TSharedPtr> Buffer; + TSharedPtr Lock; + + FSSEReceiveArchive(TSharedPtr> InBuffer, TSharedPtr InLock) + : Buffer(MoveTemp(InBuffer)), Lock(MoveTemp(InLock)) + { + SetIsSaving(true); + } + + void Serialize(void* V, int64 Length) override + { + FScopeLock ScopeLock(Lock.Get()); + Buffer->Append(static_cast(V), static_cast(Length)); + } + + FString GetArchiveName() const override { return TEXT("FSSEReceiveArchive"); } +}; + +// --------------------------------------------------------------------------- +// Lifecycle +// --------------------------------------------------------------------------- + +void UAvatarCoreAIOpenRouter::InitAIManagerChild(UAIBaseConfig* AIConfig, AActor* InWorldReferenceActor) +{ + OpenRouterConfig = Cast(AIConfig); + if (!OpenRouterConfig) + { + BroadcastAIError(TEXT("Cannot cast config to UOpenRouterConfig"), EAvatarCoreAIError::InvalidConfig); + return; + } + ActivateAI(); +} + +void UAvatarCoreAIOpenRouter::ActivateAI() +{ + SetNewState(EAvatarCoreAIState::Ready); + OnAIReady(); +} + +void UAvatarCoreAIOpenRouter::DeactivateAI() +{ + CancelActiveRequest(); + UAIBaseManager::DeactivateAI(); + SetNewState(EAvatarCoreAIState::Disconnected); +} + +void UAvatarCoreAIOpenRouter::UpdateSession() +{ + if (!OpenRouterConfig) + { + BroadcastAIError(TEXT("OpenRouterConfig is null in UpdateSession"), EAvatarCoreAIError::InvalidConfig); + return; + } + OnAIReady(); +} + +void UAvatarCoreAIOpenRouter::ClearAI() +{ + CancelActiveRequest(); + ResetSSEState(); + UAIBaseManager::ClearAI(); +} + +void UAvatarCoreAIOpenRouter::CancelActiveRequest() +{ + if (ActiveRequest.IsValid()) + { + BroadcastAILog(TEXT("OpenRouter: cancelling active HTTP request.")); + ActiveRequest->CancelRequest(); + ActiveRequest.Reset(); + } +} + +// --------------------------------------------------------------------------- +// Send +// --------------------------------------------------------------------------- + +void UAvatarCoreAIOpenRouter::SendResponseChild(FAIMessage Message, bool NotifyDelay) +{ + if (!Message.bTriggerResponse) + return; + UAvatarCoreAIOpenRouter::SetNewState(EAvatarCoreAIState::Processing); + // Cancel any stale request (safety net; queue should normally prevent overlaps) + CancelActiveRequest(); + ResetSSEState(); + SendChatCompletionRequest(Message); +} + +void UAvatarCoreAIOpenRouter::ResetSSEState() +{ + SSEStreamBufferPtr.Reset(); + SSEBufferLock.Reset(); + SSERawBuffer.Empty(); + SSEByteOffset = 0; + bResponseComplete = false; + ToolCallNameMap.Empty(); + ToolCallArgsMap.Empty(); + ToolCallIdMap.Empty(); +} + +void UAvatarCoreAIOpenRouter::SendChatCompletionRequest(FAIMessage CurrentMessage) +{ + if (!OpenRouterConfig) + { + BroadcastAIError(TEXT("OpenRouterConfig is null"), EAvatarCoreAIError::InvalidConfig); + return; + } + + // Build request body + TSharedPtr Body = MakeShareable(new FJsonObject); + Body->SetStringField(TEXT("model"), OpenRouterConfig->OpenRouterSettings.BaseAISettings.ModelID); + Body->SetBoolField(TEXT("stream"), true); + Body->SetNumberField(TEXT("max_tokens"), OpenRouterConfig->GlobalAISettings.MaxTokens); + Body->SetNumberField(TEXT("temperature"), OpenRouterConfig->GlobalAISettings.Temperature); + + Body->SetArrayField(TEXT("messages"), BuildMessagesArray(CurrentMessage)); + + TArray> Tools; + if (OpenRouterConfig->OpenRouterSettings.bSendTools) + Tools = BuildToolsArray(); + if (Tools.Num() > 0) + { + Body->SetArrayField(TEXT("tools"), Tools); + Body->SetStringField(TEXT("tool_choice"), TEXT("auto")); + } + + FString BodyString; + TSharedRef> Writer = TJsonWriterFactory<>::Create(&BodyString); + FJsonSerializer::Serialize(Body.ToSharedRef(), Writer); + + BroadcastAILog(FString::Printf(TEXT("OpenRouter request to %s/%s (tools: %d)"), + *OpenRouterConfig->OpenRouterSettings.BaseURL, *OpenRouterConfig->OpenRouterSettings.BaseAISettings.ModelID, Tools.Num())); + BroadcastAILog(BodyString, false, true); + + // Create HTTP request + ActiveRequest = FHttpModule::Get().CreateRequest(); + ActiveRequest->SetURL(OpenRouterConfig->OpenRouterSettings.BaseURL + TEXT("/chat/completions")); + ActiveRequest->SetVerb(TEXT("POST")); + ActiveRequest->SetHeader(TEXT("Content-Type"), TEXT("application/json")); + ActiveRequest->SetHeader(TEXT("Authorization"), TEXT("Bearer ") + OpenRouterConfig->OpenRouterSettings.BaseAISettings.APIKey); + if (!OpenRouterConfig->OpenRouterSettings.SiteURL.IsEmpty()) + ActiveRequest->SetHeader(TEXT("HTTP-Referer"), OpenRouterConfig->OpenRouterSettings.SiteURL); + if (!OpenRouterConfig->OpenRouterSettings.AppName.IsEmpty()) + ActiveRequest->SetHeader(TEXT("X-Title"), OpenRouterConfig->OpenRouterSettings.AppName); + ActiveRequest->SetContentAsString(BodyString); + + // Attach a receive-stream archive so the HTTP module writes bytes directly into our buffer. + // This avoids calling GetContent() on an in-progress request, which logs "Payload is incomplete". + // The lock guards against the HTTP thread (Serialize) racing the game thread (OnResponseProgress). + SSEStreamBufferPtr = MakeShared>(); + SSEBufferLock = MakeShared(); + ActiveRequest->SetResponseBodyReceiveStream(MakeShareable(new FSSEReceiveArchive(SSEStreamBufferPtr, SSEBufferLock))); + + ActiveRequest->OnRequestProgress64().BindUObject(this, &UAvatarCoreAIOpenRouter::OnResponseProgress); + ActiveRequest->OnProcessRequestComplete().BindUObject(this, &UAvatarCoreAIOpenRouter::OnRequestComplete); + + ResetRequestTimeout(); + SetNewState(EAvatarCoreAIState::Processing); + + ActiveRequest->ProcessRequest(); +} + +// --------------------------------------------------------------------------- +// Message / Tool array builders +// --------------------------------------------------------------------------- + +TArray> UAvatarCoreAIOpenRouter::BuildMessagesArray(FAIMessage CurrentMessage) +{ + TArray> Messages; + + // System instructions — always the most recent, sent as first message + FString SystemPrompt = GetSystemInstructionPromptString(false); + if (!SystemPrompt.IsEmpty()) + { + TSharedPtr SysMsg = MakeShareable(new FJsonObject); + SysMsg->SetStringField(TEXT("role"), TEXT("system")); + SysMsg->SetStringField(TEXT("content"), SystemPrompt); + Messages.Add(MakeShareable(new FJsonValueObject(SysMsg))); + } + + // Validate history: remove any assistant tool_calls entry that has no matching tool result, + // and any tool result that has no preceding assistant tool_calls. Either orphan causes a 400. + TArray History = GetAllPreviousMessage(); + for (int32 i = History.Num() - 1; i >= 0; --i) + { + const FAIMessage& Msg = History[i]; + if (Msg.Role == EAvatarCoreAIPromptRole::Assistant && !Msg.Id.IsEmpty()) + { + bool bHasResult = false; + for (int32 j = i + 1; j < History.Num(); ++j) + if (History[j].Role == EAvatarCoreAIPromptRole::Tool && History[j].Id == Msg.Id) + { bHasResult = true; break; } + // CurrentMessage may be the tool result that hasn't been added to PreviousMessages yet + if (!bHasResult && CurrentMessage.Role == EAvatarCoreAIPromptRole::Tool && CurrentMessage.Id == Msg.Id) + bHasResult = true; + if (!bHasResult) + { + BroadcastAILog(FString::Printf(TEXT("OpenRouter: dropping orphaned tool_calls entry (id=%s) from history"), *Msg.Id), true); + History.RemoveAt(i); + } + } + else if (Msg.Role == EAvatarCoreAIPromptRole::Tool) + { + bool bHasCall = false; + for (int32 j = 0; j < i; ++j) + if (History[j].Role == EAvatarCoreAIPromptRole::Assistant && History[j].Id == Msg.Id) + { bHasCall = true; break; } + if (!bHasCall) + { + BroadcastAILog(FString::Printf(TEXT("OpenRouter: dropping orphaned tool result (id=%s) from history"), *Msg.Id), true); + History.RemoveAt(i); + } + } + } + + // Conversation history (does not include CurrentMessage — it was not yet added to PreviousMessages) + auto AppendMessage = [&](const FAIMessage& Msg) + { + TSharedPtr MsgObj = MakeShareable(new FJsonObject); + + if (Msg.Role == EAvatarCoreAIPromptRole::Tool) + { + MsgObj->SetStringField(TEXT("role"), TEXT("tool")); + MsgObj->SetStringField(TEXT("tool_call_id"), Msg.Id); + MsgObj->SetStringField(TEXT("content"), Msg.Message); + } + else if (Msg.Role == EAvatarCoreAIPromptRole::Assistant && !Msg.Id.IsEmpty()) + { + // Assistant message that triggered a tool call; Message contains the tool_calls JSON array. + // content must be null (not "") when tool_calls is present — OpenAI spec requirement. + MsgObj->SetStringField(TEXT("role"), TEXT("assistant")); + MsgObj->SetField(TEXT("content"), MakeShareable(new FJsonValueNull())); + + TArray> ToolCallsArray; + TSharedRef> TCReader = TJsonReaderFactory<>::Create(Msg.Message); + if (FJsonSerializer::Deserialize(TCReader, ToolCallsArray)) + { + MsgObj->SetArrayField(TEXT("tool_calls"), ToolCallsArray); + } + else + { + BroadcastAILog(FString::Printf(TEXT("OpenRouter: failed to parse stored tool_calls JSON: %s"), *Msg.Message)); + return; + } + } + else + { + FString RoleStr; + switch (Msg.Role) + { + case EAvatarCoreAIPromptRole::User: RoleStr = TEXT("user"); break; + case EAvatarCoreAIPromptRole::Assistant: RoleStr = TEXT("assistant"); break; + case EAvatarCoreAIPromptRole::System: RoleStr = TEXT("system"); break; + default: RoleStr = TEXT("user"); break; + } + MsgObj->SetStringField(TEXT("role"), RoleStr); + MsgObj->SetStringField(TEXT("content"), Msg.Message); + } + + Messages.Add(MakeShareable(new FJsonValueObject(MsgObj))); + }; + + for (const FAIMessage& Msg : History) + AppendMessage(Msg); + + // Current message appended last — ensures it is always the newest entry + AppendMessage(CurrentMessage); + + return Messages; +} + +TArray> UAvatarCoreAIOpenRouter::BuildToolsArray() +{ + TArray> ToolsArray; + for (const FMCPToolInfo& Command : GetAvailableCommands()) + { + if (Command.Name.IsEmpty()) + continue; + + TSharedPtr FunctionObj = MakeShareable(new FJsonObject); + FunctionObj->SetStringField(TEXT("name"), Command.Name.Left(64)); + if (!Command.Description.IsEmpty()) + FunctionObj->SetStringField(TEXT("description"), Command.Description); + + if (!Command.InputScheme.IsEmpty()) + { + TSharedPtr ParamsObj; + TSharedRef> Reader = TJsonReaderFactory<>::Create(Command.InputScheme); + if (FJsonSerializer::Deserialize(Reader, ParamsObj) && ParamsObj.IsValid()) + FunctionObj->SetObjectField(TEXT("parameters"), ParamsObj); + else + BroadcastAIError(FString::Printf(TEXT("InputScheme of '%s' is not valid JSON"), *Command.Name), EAvatarCoreAIError::MCPError); + } + + TSharedPtr ToolObj = MakeShareable(new FJsonObject); + ToolObj->SetStringField(TEXT("type"), TEXT("function")); + ToolObj->SetObjectField(TEXT("function"), FunctionObj); + + ToolsArray.Add(MakeShareable(new FJsonValueObject(ToolObj))); + } + return ToolsArray; +} + +// --------------------------------------------------------------------------- +// SSE streaming +// --------------------------------------------------------------------------- + +void UAvatarCoreAIOpenRouter::OnResponseProgress(FHttpRequestPtr Request, uint64 BytesSent, uint64 BytesReceived) +{ + // Take local shared-ptr copies first. If ResetSSEState fires inside ParseSSELine + // (same call stack, game thread) it will null the members, but our locals keep the + // objects alive and the lock valid for the duration of this callback. + TSharedPtr> Buffer = SSEStreamBufferPtr; + TSharedPtr Lock = SSEBufferLock; + if (!Buffer.IsValid() || !Lock.IsValid()) + return; + + // Lock only for the brief copy of newly-arrived bytes. + // Holding the lock any longer would block the HTTP thread unnecessarily. + TArray NewBytes; + { + FScopeLock ScopeLock(Lock.Get()); + const int32 Total = Buffer->Num(); + const int32 ToRead = Total - SSEByteOffset; + if (ToRead <= 0) + return; + NewBytes.Append(Buffer->GetData() + SSEByteOffset, ToRead); + SSEByteOffset = Total; + } + + // Append only the newly arrived bytes as raw bytes — never convert to ANSI here, + // because UTF-8 multibyte sequences may straddle callback boundaries. + SSERawBuffer.Append(NewBytes); + + // Scan for complete lines. '\n' (0x0A) is never a continuation byte in UTF-8, + // so this scan is safe on raw UTF-8 data. + int32 StartPos = 0; + const int32 BufferSize = SSERawBuffer.Num(); + for (int32 i = 0; i < BufferSize; ++i) + { + if (SSERawBuffer[i] == '\n') + { + int32 End = i; + if (End > StartPos && SSERawBuffer[End - 1] == '\r') + --End; // strip \r + + if (End > StartPos) + { + // Null-terminate the slice and convert UTF-8 → FString + TArray Slice(SSERawBuffer.GetData() + StartPos, End - StartPos); + Slice.Add(0); + ParseSSELine(UTF8_TO_TCHAR((ANSICHAR*)Slice.GetData())); + } + StartPos = i + 1; + + // ParseSSELine may have triggered ResetSSEState (e.g. via OnAIReady → SendResponseChild). + // If the buffer was cleared, stop processing stale data. + if (SSERawBuffer.Num() == 0) + return; + } + } + + // Discard all processed bytes; keep the partial trailing line in the buffer. + // Guard against the buffer being reset by a ParseSSELine side-effect. + if (StartPos > 0 && StartPos <= SSERawBuffer.Num()) + SSERawBuffer.RemoveAt(0, StartPos); +} + +void UAvatarCoreAIOpenRouter::ParseSSELine(const FString& Line) +{ + // SSE lines look like: "data: {...}" or "data: [DONE]" + if (!Line.StartsWith(TEXT("data: "))) + return; + + FString Data = Line.Mid(6); // strip "data: " + + if (Data == TEXT("[DONE]")) + return; + + TSharedPtr JsonObj; + TSharedRef> Reader = TJsonReaderFactory<>::Create(Data); + if (!FJsonSerializer::Deserialize(Reader, JsonObj) || !JsonObj.IsValid()) + { + BroadcastAILog(FString::Printf(TEXT("OpenRouter: failed to parse SSE chunk: %s"), *Data), false, true); + return; + } + + const TArray>* Choices; + if (!JsonObj->TryGetArrayField(TEXT("choices"), Choices) || Choices->Num() == 0) + return; + + TSharedPtr Choice = (*Choices)[0]->AsObject(); + if (!Choice.IsValid()) + return; + + TSharedPtr Delta = Choice->GetObjectField(TEXT("delta")); + + // Text content chunk + FString ContentChunk; + if (Delta.IsValid() && Delta->TryGetStringField(TEXT("content"), ContentChunk) && !ContentChunk.IsEmpty()) + { + ClearRequestTimeout(); + OnAIResponse(ContentChunk, false); + } + + // Tool call argument accumulation + const TArray>* ToolCallsArr; + if (Delta.IsValid() && Delta->TryGetArrayField(TEXT("tool_calls"), ToolCallsArr)) + { + for (const TSharedPtr& TCValue : *ToolCallsArr) + { + TSharedPtr TC = TCValue->AsObject(); + if (!TC.IsValid()) continue; + + int32 Idx = 0; + TC->TryGetNumberField(TEXT("index"), Idx); + + // Name and id only arrive on the first delta for this index + FString TCName, TCId; + TSharedPtr FuncObj = TC->GetObjectField(TEXT("function")); + if (FuncObj.IsValid()) + { + FuncObj->TryGetStringField(TEXT("name"), TCName); + if (!TCName.IsEmpty()) + ToolCallNameMap.FindOrAdd(Idx) += TCName; + + FString ArgsDelta; + if (FuncObj->TryGetStringField(TEXT("arguments"), ArgsDelta)) + ToolCallArgsMap.FindOrAdd(Idx) += ArgsDelta; + } + if (TC->TryGetStringField(TEXT("id"), TCId) && !TCId.IsEmpty()) + ToolCallIdMap.FindOrAdd(Idx) = TCId; + } + } + + // Finish reason + FString FinishReason; + Choice->TryGetStringField(TEXT("finish_reason"), FinishReason); + + if (FinishReason == TEXT("stop") && !bResponseComplete) + { + bResponseComplete = true; + ClearRequestTimeout(); + ActiveRequest.Reset(); + OnAIResponse(TEXT(""), true); // IsFinal — base adds assistant message to history + SetNewState(EAvatarCoreAIState::Ready); + OnAIReady(); // drains ResponseQueue + } + else if (FinishReason == TEXT("tool_calls") && !bResponseComplete) + { + bResponseComplete = true; + ClearRequestTimeout(); + ActiveRequest.Reset(); + HandleToolCallsDone(); + } +} + +void UAvatarCoreAIOpenRouter::HandleToolCallsDone() +{ + // First version: handle the tool call at index 0 (same single-call behaviour as Realtime) + if (!ToolCallIdMap.Contains(0) || !ToolCallNameMap.Contains(0)) + { + BroadcastAIError(TEXT("OpenRouter: tool_calls done but no accumulated call data at index 0"), + EAvatarCoreAIError::FunctionCallFailed); + SetNewState(EAvatarCoreAIState::Error); + return; + } + + FString CallId = ToolCallIdMap[0]; + FString CallName = ToolCallNameMap[0]; + FString CallArgs = ToolCallArgsMap.FindRef(0); + + BroadcastAILog(FString::Printf(TEXT("OpenRouter: tool call '%s' (id=%s) args=%s"), *CallName, *CallId, *CallArgs), true); + + // Build the tool_calls JSON array to persist in message history so the next + // request has the correct assistant→tool context. + TSharedPtr FuncObj = MakeShareable(new FJsonObject); + FuncObj->SetStringField(TEXT("name"), CallName); + FuncObj->SetStringField(TEXT("arguments"), CallArgs); + + TSharedPtr ToolCallObj = MakeShareable(new FJsonObject); + ToolCallObj->SetStringField(TEXT("id"), CallId); + ToolCallObj->SetStringField(TEXT("type"), TEXT("function")); + ToolCallObj->SetObjectField(TEXT("function"), FuncObj); + + TArray> ToolCallsArray; + ToolCallsArray.Add(MakeShareable(new FJsonValueObject(ToolCallObj))); + + FString ToolCallsJson; + TSharedRef> TCWriter = TJsonWriterFactory<>::Create(&ToolCallsJson); + FJsonSerializer::Serialize(ToolCallsArray, TCWriter); + + // Add assistant message with tool_calls to history + FAIMessage AssistantToolCall; + AssistantToolCall.Role = EAvatarCoreAIPromptRole::Assistant; + AssistantToolCall.Id = CallId; + AssistantToolCall.Message = ToolCallsJson; // BuildMessagesArray reads this back + AddMessageToArray(AssistantToolCall); + + // RunMCPCommand sets GettingInfo state; base propagates CallId → CommandFinished → Tool role result + RunMCPCommand(CallName, CallArgs, CallId); +} + +// --------------------------------------------------------------------------- +// HTTP completion handler +// --------------------------------------------------------------------------- + +void UAvatarCoreAIOpenRouter::OnRequestComplete(FHttpRequestPtr Request, FHttpResponsePtr Response, bool bSuccess) +{ + ActiveRequest.Reset(); + + // Successful streaming already handled by ParseSSELine. + // This callback is for error cases only. + if (bSuccess && Response.IsValid() && Response->GetResponseCode() < 300) + return; + + ClearRequestTimeout(); + + int32 Code = Response.IsValid() ? Response->GetResponseCode() : 0; + // GetContentAsString() is empty when SetResponseBodyReceiveStream is active — + // all bytes (including error bodies) were written to SSEStreamBufferPtr. + FString Body; + if (Response.IsValid()) + { + Body = Response->GetContentAsString(); + if (Body.IsEmpty() && SSEStreamBufferPtr.IsValid() && SSEStreamBufferPtr->Num() > 0) + { + TArray BodyBytes = *SSEStreamBufferPtr; + BodyBytes.Add(0); + Body = UTF8_TO_TCHAR((ANSICHAR*)BodyBytes.GetData()); + } + } + + // 422 with a tools-related body means this model doesn't support OpenAI-compatible + // function calling. Disable bSendTools in the config for this model. + if (Code == 422 && Body.Contains(TEXT("tools"))) + { + BroadcastAIError( + FString::Printf(TEXT("OpenRouter 422: model '%s' rejected the tools format. " + "Disable bSendTools in OpenRouterConfig for models that don't support " + "OpenAI-compatible function calling. Raw: %s"), *OpenRouterConfig->OpenRouterSettings.BaseAISettings.ModelID, *Body), + EAvatarCoreAIError::NetworkError); + } + else + { + BroadcastAIError( + FString::Printf(TEXT("OpenRouter HTTP error %d: %s"), Code, *Body), + EAvatarCoreAIError::NetworkError); + } + SetNewState(EAvatarCoreAIState::Error); +} diff --git a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/RealtimeAPI/AvatarCoreAIRealtime.cpp b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/RealtimeAPI/AvatarCoreAIRealtime.cpp index d138830..01ebe5a 100644 --- a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/RealtimeAPI/AvatarCoreAIRealtime.cpp +++ b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/RealtimeAPI/AvatarCoreAIRealtime.cpp @@ -37,9 +37,9 @@ void UAvatarCoreAIRealtime::DeactivateAI() UAvatarCoreAIRealtime::DisconnectFromWebSocket(); } -void UAvatarCoreAIRealtime::SendResponseChild(FAIMessage Message, bool NotifyDelay, bool TriggerResponse) +void UAvatarCoreAIRealtime::SendResponseChild(FAIMessage Message, bool NotifyDelay) { - UAvatarCoreAIRealtime::CreateConversationItem(Message, TriggerResponse); + UAvatarCoreAIRealtime::CreateConversationItem(Message); } void UAvatarCoreAIRealtime::ClearAI() @@ -51,7 +51,9 @@ void UAvatarCoreAIRealtime::ClearAI() } void UAvatarCoreAIRealtime::UpdateSession() -{ +{ + UAvatarCoreAIRealtime::SetNewState(EAvatarCoreAIState::Initializing); + if (!RealtimeConfig) { BroadcastAIError("RealtimeConfig is null in UpdateSession", EAvatarCoreAIError::InvalidConfig); return; @@ -61,7 +63,7 @@ void UAvatarCoreAIRealtime::UpdateSession() TArray Modalities; // Voice as string - FString VoiceStr = StaticEnum()->GetNameStringByValue((int64)RealtimeConfig->Voice).ToLower(); + FString VoiceStr = StaticEnum()->GetNameStringByValue((int64)RealtimeConfig->RealtimeSettings.Voice).ToLower(); if (VoiceStr == TEXT("undefined")) VoiceStr = TEXT(""); // Build session object @@ -69,7 +71,7 @@ void UAvatarCoreAIRealtime::UpdateSession() FString InstructionsString = UAIBaseManager::GetSystemInstructionPromptString(false); - if (RealtimeConfig->AIModelAudioOutput) + if (RealtimeConfig->GlobalAISettings.AIModelAudioOutput) Modalities.Add(TEXT("audio")); else Modalities.Add(TEXT("text")); @@ -84,7 +86,7 @@ void UAvatarCoreAIRealtime::UpdateSession() SessionObj->SetStringField(TEXT("type"), TEXT("realtime")); - SessionObj->SetNumberField(TEXT("max_output_tokens"), RealtimeConfig->MaxTokens); + SessionObj->SetNumberField(TEXT("max_output_tokens"), RealtimeConfig->GlobalAISettings.MaxTokens); // Add available tools as functions to the session JSON TArray Commands = GetAvailableCommands(); @@ -130,7 +132,7 @@ void UAvatarCoreAIRealtime::UpdateSession() TSharedPtr AudioObj = MakeShareable(new FJsonObject); SessionObj->SetObjectField(TEXT("audio"), AudioObj); - if (RealtimeConfig->InputAudioStreaming) { + if (RealtimeConfig->RealtimeSettings.InputAudioStreaming) { TSharedPtr AudioInputObj = MakeShareable(new FJsonObject); AudioObj->SetObjectField(TEXT("input"), AudioInputObj); @@ -158,7 +160,7 @@ void UAvatarCoreAIRealtime::UpdateSession() } - if (RealtimeConfig->AIModelAudioOutput) { + if (RealtimeConfig->GlobalAISettings.AIModelAudioOutput) { if (!VoiceStr.IsEmpty()) { TSharedPtr AudioOutputObj = MakeShareable(new FJsonObject); AudioObj->SetObjectField(TEXT("output"), AudioOutputObj); @@ -186,18 +188,18 @@ void UAvatarCoreAIRealtime::UpdateSession() void UAvatarCoreAIRealtime::ConnectToWebSocket() { - FString ServerURL = TEXT("wss://" + RealtimeConfig->BaseURL + "/v1/realtime?model=" + RealtimeConfig->Model); + FString ServerURL = TEXT("wss://" + RealtimeConfig->RealtimeSettings.BaseURL + "/v1/realtime?model=" + RealtimeConfig->RealtimeSettings.BaseAISettings.ModelID); BroadcastAILog(FString::Printf(TEXT("OpenAI ServerURL: %s"), *ServerURL)); FString ServerProtocol = TEXT(""); // Set up headers for authentication TMap Headers; - if(RealtimeConfig->IsAzureOpenAI) - Headers.Add(TEXT("api-key"), *RealtimeConfig->APIKey); + if(RealtimeConfig->RealtimeSettings.IsAzureOpenAI) + Headers.Add(TEXT("api-key"), *RealtimeConfig->RealtimeSettings.BaseAISettings.APIKey); else { - Headers.Add(TEXT("Authorization"), TEXT("Bearer " + RealtimeConfig->APIKey)); + Headers.Add(TEXT("Authorization"), TEXT("Bearer " + RealtimeConfig->RealtimeSettings.BaseAISettings.APIKey)); } @@ -254,7 +256,7 @@ void UAvatarCoreAIRealtime::WebSocketSendType(const FString& type) // Serialize the JSON object if (FJsonSerializer::Serialize(JsonObject.ToSharedRef(), Writer)) { - BroadcastAILog(FString::Printf(TEXT("Sending Type: %s"), *OutputString), false); + BroadcastAILog(FString::Printf(TEXT("Sending Type: %s"), *OutputString), false, true); UAvatarCoreAIRealtime::WebSocketSend(OutputString); } else @@ -264,40 +266,62 @@ void UAvatarCoreAIRealtime::WebSocketSendType(const FString& type) } } -void UAvatarCoreAIRealtime::CreateConversationItem(FAIMessage Message, bool triggerResponse) +void UAvatarCoreAIRealtime::CreateConversationItem(FAIMessage Message) { TSharedPtr RootObject = MakeShareable(new FJsonObject); RootObject->SetStringField("type", "conversation.item.create"); - - // Create the item object + TSharedPtr ItemObject = MakeShareable(new FJsonObject); + + // Tool result — map to Realtime API's function_call_output item type + if (Message.Role == EAvatarCoreAIPromptRole::Tool) + { + ItemObject->SetStringField("type", "function_call_output"); + ItemObject->SetStringField("call_id", Message.Id); + ItemObject->SetStringField("output", Message.Message); + RootObject->SetObjectField("item", ItemObject); + + FString OutputString; + TSharedRef>> Writer = + TJsonWriterFactory>::Create(&OutputString); + FJsonSerializer::Serialize(RootObject.ToSharedRef(), Writer); + UAvatarCoreAIRealtime::WebSocketSend(OutputString); + + if (Message.bTriggerResponse) + UAvatarCoreAIRealtime::CreateReseponse(); + return; + } + ItemObject->SetStringField("type", "message"); ItemObject->SetStringField("role", UAvatarCoreAIRealtime::GetRoleAsString(Message.Role)); - // Create the content array with an audio object inside + FString ContentType = "input_text"; + if (Message.Role == EAvatarCoreAIPromptRole::Assistant) + { + ContentType = "output_text"; + } + TArray> ContentArray; + TSharedPtr TextObject = MakeShareable(new FJsonObject); + TextObject->SetStringField("type", ContentType); + TextObject->SetStringField("text", Message.Message); + ContentArray.Add(MakeShareable(new FJsonValueObject(TextObject))); - TSharedPtr AudioObject = MakeShareable(new FJsonObject); - if(Message.Role == EAvatarCoreAIPromptRole::User || triggerResponse) - AudioObject->SetStringField("type", "input_text"); - else - AudioObject->SetStringField("type", "output_text"); - AudioObject->SetStringField("text", Message.Message); - ContentArray.Add(MakeShareable(new FJsonValueObject(AudioObject))); ItemObject->SetArrayField("content", ContentArray); - - // Add item to the root object RootObject->SetObjectField("item", ItemObject); - // Convert the root JSON object to a string + // Optional but recommended when replaying history: + // RootObject->SetStringField("previous_item_id", LastKnownItemId); + FString OutputString; TSharedRef>> Writer = TJsonWriterFactory>::Create(&OutputString); FJsonSerializer::Serialize(RootObject.ToSharedRef(), Writer); - UAvatarCoreAIRealtime::WebSocketSend(OutputString); //Send the Message + UAvatarCoreAIRealtime::WebSocketSend(OutputString); - if (triggerResponse) { + if (Message.bTriggerResponse) + { UAvatarCoreAIRealtime::CreateReseponse(); } } @@ -316,7 +340,7 @@ void UAvatarCoreAIRealtime::CreateReseponse() // Create the content array with an audio object inside TArray> ModalitiesArray; //There is an error if we submit both text and audio; nevertheless text is always included in audio mode - if(RealtimeConfig->AIModelAudioOutput) + if(RealtimeConfig->GlobalAISettings.AIModelAudioOutput) ModalitiesArray.Add(MakeShareable(new FJsonValueString("audio"))); else ModalitiesArray.Add(MakeShareable(new FJsonValueString("text"))); @@ -504,7 +528,7 @@ void UAvatarCoreAIRealtime::OnWebSocketConnectionStringReceived(const FString& M JsonObject->TryGetStringField(TEXT("delta"), DeltaResponse.delta); float CurrentRequestDuration = (FDateTime::Now() - CurrentRequestStartTime).GetTotalSeconds(); - BroadcastAILog(FString::Printf(TEXT("Response of type %s after %f seconds"), *TypeString, CurrentRequestDuration)); + BroadcastAILog(FString::Printf(TEXT("Response of type %s after %f seconds"), *TypeString, CurrentRequestDuration), false, true); if (DeltaResponse.response_id != CurrentRequestID) //Has been cancelled and cleared { @@ -526,13 +550,13 @@ void UAvatarCoreAIRealtime::OnWebSocketConnectionStringReceived(const FString& M } //float CurrentRequestDuration = (FDateTime::Now() - CurrentRequestStartTime).GetTotalSeconds(); - if (CurrentRequestDuration < 0.5f && !functionCallRunning) + if (CurrentRequestDuration < 0.25f && !functionCallRunning && !CurrentRequestID.IsEmpty()) { CurrentRequestID.Empty(); if (CurrentRetries < MaxRetries) { CurrentRetries++; - BroadcastAILog(FString::Printf(TEXT("Response.done is way to fast. Something is fishy. Let's try again OpenAI! %s"), *TypeString), true); + BroadcastAILog(FString::Printf(TEXT("Response.done is way to fast. Something is fishy %s"), *TypeString), true); CreateReseponse(); } else { @@ -543,7 +567,7 @@ void UAvatarCoreAIRealtime::OnWebSocketConnectionStringReceived(const FString& M } } - if (RealtimeConfig->AIModelAudioOutput && ResponseAudioDone && ResponseTextDone || !RealtimeConfig->AIModelAudioOutput && ResponseTextDone) + if (RealtimeConfig->GlobalAISettings.AIModelAudioOutput && ResponseAudioDone && ResponseTextDone || !RealtimeConfig->GlobalAISettings.AIModelAudioOutput && ResponseTextDone) { ClearRequestTimeout(); RequestState = EOpenAIRequestState::done; @@ -592,7 +616,7 @@ void UAvatarCoreAIRealtime::OnWebSocketConnectionStringReceived(const FString& M // Resize and directly copy raw PCM data into PCMData PCMData.AddUninitialized(Size); FBase64::Decode(DeltaResponse.delta, PCMData); - BroadcastAILog(FString::Printf(TEXT("Adding: %s"), *TypeString)); + BroadcastAILog(FString::Printf(TEXT("Adding: %s"), *TypeString), false, true); OnAudioChunk.Broadcast(PCMData, false); } } @@ -651,7 +675,7 @@ void UAvatarCoreAIRealtime::OnWebSocketConnectionDropped(int32 StatusCode, const void UAvatarCoreAIRealtime::SetOpenAIAudioOutput(bool InAudioOutput) { - RealtimeConfig->AIModelAudioOutput = InAudioOutput; + RealtimeConfig->GlobalAISettings.AIModelAudioOutput = InAudioOutput; } void UAvatarCoreAIRealtime::OnSTTAudioChunk(TArray AudioChunks) diff --git a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/AIBaseConfig.h b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/AIBaseConfig.h index 6a2eeb2..a94c06b 100644 --- a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/AIBaseConfig.h +++ b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/AIBaseConfig.h @@ -6,27 +6,28 @@ #include "MCP/MCPUnrealCommand.h" #include "MCP/MCPBaseConfig.h" #include "UObject/NoExportTypes.h" +#include "AvatarCoreAIEnumsAndStructs.h" #include "AIBaseConfig.generated.h" class UMCPBaseManager; class UAIBaseManager; -/** - * - */ -UCLASS(Abstract, Blueprintable, BlueprintType) -class AVATARCORE_AI_API UAIBaseConfig : public UObject +//These are still base settings but separate so that the global settings can be applied for various AI providers +USTRUCT(BlueprintType) +struct FBaseAISettings { GENERATED_BODY() -public: - - //Class of the Manager - UPROPERTY(BlueprintReadOnly, Category = "AvatarCoreAI|Base") - TSubclassOf AIManagerClass; + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Settings", meta = (ExposeOnSpawn = "true")) + FString APIKey = ""; - // All those neat little system prompts that make our avatars sooo great UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Settings", meta = (ExposeOnSpawn = "true")) - TArray SystemPrompts; + FString ModelID = ""; +}; + +USTRUCT(BlueprintType) +struct FGlobalAISettings +{ + GENERATED_BODY() // Check user transcription for inappropriate behaviour first (adds a delay!) UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|MCP", meta = (ExposeOnSpawn = "true")) @@ -36,22 +37,16 @@ public: UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|MCP", meta = (ExposeOnSpawn = "true")) bool bUseMCPServer = true; - // Class of the MCP Server - UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|MCP", meta = (ExposeOnSpawn = "true")) - TSubclassOf MCPManagerClass; - - // Config of the MCP Server - UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|MCP", meta = (ExposeOnSpawn = "true")) - UMCPBaseConfig* MCPConfig; - - // Array of Unreal command objects to be used by this config - UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|MCP", meta = (ExposeOnSpawn = "true")) - TArray> UnrealCommands; - // Does the AI model generate Audio Chunks that can be forwarded to the TTS Manager? UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Settings", meta = (ExposeOnSpawn = "true")) bool AIModelAudioOutput = false; + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Settings", meta = (ExposeOnSpawn = "true")) + int32 MaxTokens = 1500; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Settings", meta = (ExposeOnSpawn = "true")) + float Temperature = 0.8f; + // Does the AI model generate Audio Chunks that can be forwarded to the TTS Manager? UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Settings", meta = (ExposeOnSpawn = "true")) int RequestTimeout = 10.0f; @@ -64,3 +59,37 @@ public: UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Settings", meta = (ExposeOnSpawn = "true")) int MaxMessages = -1; }; + +/** + * + */ +UCLASS(Abstract, Blueprintable, BlueprintType) +class AVATARCORE_AI_API UAIBaseConfig : public UObject +{ + GENERATED_BODY() + +public: + + //Class of the Manager + UPROPERTY(BlueprintReadOnly, Category = "AvatarCoreAI|Settings", meta = (ExposeOnSpawn = "true")) + TSubclassOf AIManagerClass; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Settings", meta = (ExposeOnSpawn = "true")) + FGlobalAISettings GlobalAISettings; + + // All those neat little system prompts that make our avatars sooo great + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Settings", meta = (ExposeOnSpawn = "true")) + TArray SystemPrompts; + + // Array of Unreal command objects to be used by this config + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|MCP", meta = (ExposeOnSpawn = "true")) + TArray> UnrealCommands; + + // Class of the MCP Server + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|MCP", meta = (ExposeOnSpawn = "true")) + TSubclassOf MCPManagerClass; + + // Config of the MCP Server + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|MCP", meta = (ExposeOnSpawn = "true")) + UMCPBaseConfig* MCPConfig; +}; diff --git a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/AIBaseManager.h b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/AIBaseManager.h index 63c5a6f..c4df633 100644 --- a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/AIBaseManager.h +++ b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/AIBaseManager.h @@ -4,15 +4,17 @@ #include "CoreMinimal.h" #include "UObject/Object.h" -#include "AIBaseConfig.h" +#include "AIBaseConfig.h" #include "AvatarCoreAIEnumsAndStructs.h" #include "MCP/MCPBaseManager.h" #include "AIBaseManager.generated.h" +class FJsonValue; + // Delegate/Event Declarations DECLARE_DYNAMIC_MULTICAST_DELEGATE_FourParams(FOnAISubtitle, FString, Chunk, FString, Answer, int, responseID, bool, IsFinal); DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FOnAIStateChanged, EAvatarCoreAIState, NewState); -DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FOnAILog, FString, Message); +DECLARE_DYNAMIC_MULTICAST_DELEGATE_TwoParams(FOnAILog, FString, Message, bool, VeryVerbose); DECLARE_DYNAMIC_MULTICAST_DELEGATE_TwoParams(FOnAIError, FString, ErrorMessage, EAvatarCoreAIError, ErrorType); DECLARE_DYNAMIC_MULTICAST_DELEGATE_TwoParams(FMulticastDelegateRealtimeAPIAudioChunk, const TArray, PCMData, bool, IsFinal); DECLARE_DYNAMIC_MULTICAST_DELEGATE(FOnAIDelayedAnswer); @@ -111,10 +113,10 @@ public: * Send Response/Question to the AI Model. If NotifyDelay is true call the DelayedAnswer Event when time defined in AIConfig has passed. */ UFUNCTION(BlueprintCallable, Category = "AvatarCoreAI") - void SendResponse(FAIMessage Message, bool NotifyDelay, bool TriggerResponse); + void SendResponse(FAIMessage Message, bool NotifyDelay); UFUNCTION(BlueprintCallable, Category = "AvatarCoreAI") - virtual void SendResponseChild(FAIMessage Message, bool NotifyDelay, bool TriggerResponse) {}; + virtual void SendResponseChild(FAIMessage Message, bool NotifyDelay) {}; /** * Make the AI Model repeat the Text. @@ -132,7 +134,7 @@ public: * Log a debug message and fire the OnAILog event on the game thread. */ UFUNCTION(BlueprintCallable, Category = "AvatarCoreAI|Debug") - virtual void BroadcastAILog(const FString& Message, bool ShowAlways = false); + virtual void BroadcastAILog(const FString& Message, bool ShowAlways = false, bool VeryVerbose = false); /** * Broadcast an AI error and fire the OnAIError event on the game thread. @@ -150,19 +152,25 @@ public: * Set system instruction by array. */ UFUNCTION(BlueprintCallable, Category = "AvatarCoreAI|System Instruction") - void AddSystemInstructions(const TArray SystemInstructions, bool WipeCurrent); + void AddSystemInstructions(const TArray SystemInstructions, bool AutoSyncWithAI); /** * Add a new system instruction by name. */ UFUNCTION(BlueprintCallable, Category = "AvatarCoreAI|System Instruction") - void AddSystemInstruction(const FName Name, const FString NewSystemInstruction, bool AddAsFirst); + void AddSystemInstruction(const FSystemInstruction SystemInstruction, bool AddAsFirst, bool AutoSyncWithAI); + + /** + * Remove a system instruction by name. + */ + UFUNCTION(BlueprintCallable, Category = "AvatarCoreAI|System Instruction") + void RemoveSystemInstruction(const FName SystemInstruction, bool AutoSyncWithAI); /** * Remove a system instruction by name. */ UFUNCTION(BlueprintCallable, Category = "AvatarCoreAI|System Instruction") - void RemoveSystemInstruction(const FName Name); + void RemoveSystemInstructions(const TArray SystemInstructions, bool AutoSyncWithAI); /** * Parse to System Prompt @@ -174,7 +182,7 @@ public: * Clear all System Instruction. */ UFUNCTION(BlueprintCallable, Category = "AvatarCoreAI|System Instruction") - void ClearAllSystemInstructios(); + void ClearAllSystemInstructions(bool AutoSyncWithAI); // Add the prompt that let the avatar repeat what we want it to say UFUNCTION(BlueprintCallable, Category = "AvatarCoreAI|System Instruction") @@ -192,9 +200,9 @@ public: UFUNCTION(BlueprintCallable, Category = "AvatarCoreAI|MCP") UMCPBaseManager* GetMCPManager(); - // Add a command at runtime (handles AddToRoot) + // Register a command class at runtime; an instance is created only when the command is invoked UFUNCTION(BlueprintCallable, Category = "AvatarCoreAI|MCP Commands") - void AddUnrealCommand(UMCPUnrealCommand* Command); + void AddUnrealCommand(TSubclassOf CommandClass); // Remove a command by name at runtime (handles RemoveFromRoot) UFUNCTION(BlueprintCallable, Category = "AvatarCoreAI|Commands") @@ -206,8 +214,9 @@ public: /** * Runs a command by name with the given payload, rebinding completion/failure events. + * ToolCallId is stored on the command instance and propagated back through CommandFinished → FAIMessage.Id. */ - void RunMCPCommand(FString CommandName, FString Payload); + void RunMCPCommand(FString CommandName, FString Payload, FString ToolCallId = TEXT("")); // Clear all running commands UFUNCTION(BlueprintCallable, Category = "AvatarCoreAI|MCP Commands") @@ -218,16 +227,21 @@ public: protected: - /** - * Called when a command finishes successfully. Override in child classes to handle result. - */ + /** Bound to UMCPUnrealCommand::OnCommandDone — message already has Role=Tool and Id set. */ UFUNCTION() - void CommandFinished(const FString& Command, const FString& Payload); - /** - * Called when a command fails. Override in child classes to handle error. - */ + void CommandFinished(const FAIMessage& Message); + + /** Bound to UMCPUnrealCommand::OnCommandFailed. */ + UFUNCTION() + void CommandFailed(const FAIMessage& Message); + + /** Bound to MCPManager::OnMCPCommandDone — constructs FAIMessage from raw strings. */ + UFUNCTION() + void MCPCommandFinished(const FString& Command, const FString& Payload); + + /** Bound to MCPManager::OnMCPCommandFailed. */ UFUNCTION() - void CommandFailed(const FString& Command, const FString& Payload); + void MCPCommandFailed(const FString& Command, const FString& Payload); //Add System/User/Assistant Message to memory archive void AddMessageToArray(FAIMessage NewMessage); @@ -235,6 +249,10 @@ protected: //Add System/User/Assistant Message to memory archive TArray GetAllPreviousMessage(); + // Builds an OpenAI-compatible messages array: system prompt + history + CurrentMessage appended last. + // Reusable by any OpenAI-compatible provider implementation. + TArray> BuildOpenAIMessagesArray(const FAIMessage& CurrentMessage); + //MCP Log Event UFUNCTION() void OnMCPLogReceived(const FString& LogMessage); @@ -259,9 +277,15 @@ protected: /** Actor used as world context for commands */ TWeakObjectPtr WorldReferenceActor; - TArray UnrealCommands; + TArray> UnrealCommandClasses; TArray UnrealCommandsToolInfos; + // Maps MCP server command name → tool_call_id for propagation through MCPCommandFinished + TMap MCPToolCallIds; + + UPROPERTY() + TArray ActiveCommands; + /** MCP Manager for FastMCP server communication */ UPROPERTY() UMCPBaseManager* MCPManager; @@ -284,9 +308,6 @@ protected: //There is a function call in progress bool functionCallRunning = false; - //System Instruction - FString SystemInstruction; - //Current State the AI Manager EAvatarCoreAIState CurrentAIState = EAvatarCoreAIState::Disconnected; diff --git a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/AvatarCoreAIEnumsAndStructs.h b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/AvatarCoreAIEnumsAndStructs.h index 7fb782c..d679e6e 100644 --- a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/AvatarCoreAIEnumsAndStructs.h +++ b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/AvatarCoreAIEnumsAndStructs.h @@ -24,6 +24,7 @@ enum class EAvatarCoreAIPromptRole : uint8 User UMETA(DisplayName = "User"), Assistant UMETA(DisplayName = "Assistant"), System UMETA(DisplayName = "System"), + Tool UMETA(DisplayName = "Tool"), }; @@ -61,6 +62,13 @@ struct FAIMessage UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AI") FString Message; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AI") + bool bTriggerResponse = true; + + // tool_call_id when Role==Tool; call_id reference when Role==Assistant with tool_calls + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AI") + FString Id; }; USTRUCT(BlueprintType) @@ -100,3 +108,4 @@ struct FMCPToolInfo OutputScheme = ""; } }; + diff --git a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/AvatarCore_AI.h b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/AvatarCore_AI.h index 41fd111..5cadd85 100644 --- a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/AvatarCore_AI.h +++ b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/AvatarCore_AI.h @@ -12,4 +12,9 @@ public: /** IModuleInterface implementation */ virtual void StartupModule() override; virtual void ShutdownModule() override; + +private: +#if WITH_EDITOR + void AddDBDirectoryToPackaging(); +#endif }; diff --git a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/MCP/FastMCP/FastMCPConfig.h b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/MCP/FastMCP/FastMCPConfig.h index 5bbac45..a9754e2 100644 --- a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/MCP/FastMCP/FastMCPConfig.h +++ b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/MCP/FastMCP/FastMCPConfig.h @@ -18,7 +18,7 @@ public: //Direction to the Script that start FastMCP UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|MCP", meta = (ExposeOnSpawn = "true")) - FString MCPExecutable = FPaths::ProjectPluginsDir() +"AvatarCore_AI/Source/ThirdParty/MCPServer/FastMCP/FastMCPServer.bat"; + FString MCPExecutable = FPaths::ProjectContentDir() +"DB/FastMCP/FastMCPServer.bat"; //Custom python environment - "python" will use the system default UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreTTS|CoquiTTS", meta = (ExposeOnSpawn = "true")) diff --git a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/MCP/MCPUnrealCommand.h b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/MCP/MCPUnrealCommand.h index 7091077..d204e60 100644 --- a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/MCP/MCPUnrealCommand.h +++ b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/MCP/MCPUnrealCommand.h @@ -5,8 +5,8 @@ #include "AvatarCoreAIEnumsAndStructs.h" #include "MCPUnrealCommand.generated.h" -DECLARE_DYNAMIC_MULTICAST_DELEGATE_TwoParams(FOnAICommandDone, const FString&, CommandName, const FString&, Payload); -DECLARE_DYNAMIC_MULTICAST_DELEGATE_TwoParams(FOnAICommandFailed, const FString&, CommandName, const FString&, Payload); +DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FOnAICommandDone, const FAIMessage&, Message); +DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FOnAICommandFailed, const FAIMessage&, Message); /** * Base class for MCP/AI commands. @@ -23,6 +23,19 @@ public: UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Command") float TimeoutSeconds = 10.0f; + // Set by AIBaseManager::RunMCPCommand; propagated back to FAIMessage.Id in CommandFinished + UPROPERTY(BlueprintReadWrite, Category = "Command") + FString Id; + + UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "Context", meta = (ExposeOnSpawn = "true")) + TObjectPtr RequiredWorldContext = nullptr; + + UFUNCTION(BlueprintCallable, Category = "Context") + void SetWorldContext(UObject* NewWorldContext); + + UFUNCTION(BlueprintCallable, Category = "Context") + UObject* GetWorldContextObject() const; + // Result event (success) UPROPERTY(BlueprintAssignable, Category = "Command") FOnAICommandDone OnCommandDone; @@ -61,13 +74,13 @@ public: UFUNCTION(BlueprintCallable, Category = "Command") FString GetCommandOutputScheme(); - /** Call this when the command is finished successfully */ + /** Call this when the command is finished successfully. Set bTriggerResponse=false to suppress the AI follow-up. */ UFUNCTION(BlueprintCallable, Category = "Command") - void FinishCommand(const FString& Payload); + void FinishCommand(const FString& Payload, bool bTriggerResponse = true); - /** Call this when the command fails (timeout or error) */ + /** Call this when the command fails (timeout or error). */ UFUNCTION(BlueprintCallable, Category = "Command") - void FailCommand(const FString& Payload); + void FailCommand(const FString& Payload, bool bTriggerResponse = true); protected: FTimerHandle TimeoutHandle; diff --git a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/OpenRouter/AvatarCoreAIOpenRouter.h b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/OpenRouter/AvatarCoreAIOpenRouter.h new file mode 100644 index 0000000..4ee8c7a --- /dev/null +++ b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/OpenRouter/AvatarCoreAIOpenRouter.h @@ -0,0 +1,74 @@ +// Fill out your copyright notice in the Description page of Project Settings. + +#pragma once + +#include "CoreMinimal.h" +#include "AIBaseManager.h" +#include "OpenRouter/OpenRouterConfig.h" +#include "Interfaces/IHttpRequest.h" +#include "Interfaces/IHttpResponse.h" +#include "AvatarCoreAIOpenRouter.generated.h" + +UCLASS(Blueprintable, BlueprintType) +class AVATARCORE_AI_API UAvatarCoreAIOpenRouter : public UAIBaseManager +{ + GENERATED_BODY() + +public: + + // UAIBaseManager overrides + void InitAIManagerChild(UAIBaseConfig* AIConfig, AActor* InWorldReferenceActor) override; + void ActivateAI() override; + void DeactivateAI() override; + void UpdateSession() override; + void SendResponseChild(FAIMessage Message, bool NotifyDelay) override; + void ClearAI() override; + + // Cancel the active HTTP request if one is in flight + UFUNCTION(BlueprintCallable, Category = "AvatarCoreAI|OpenRouter") + void CancelActiveRequest(); + +private: + + UOpenRouterConfig* OpenRouterConfig = nullptr; + + TSharedPtr ActiveRequest; + + // Per-request HTTP receive buffer. FSSEReceiveArchive appends bytes here directly, + // so we never call GetContent() on a live request (avoids "Payload is incomplete" warnings). + // A new TArray is created for each request so old and new request data are isolated. + TSharedPtr> SSEStreamBufferPtr; + + // Guards concurrent access: HTTP thread writes via FSSEReceiveArchive::Serialize, + // game thread reads in OnResponseProgress. Both hold shared refs so neither + // outlives the lock even if ResetSSEState fires mid-callback. + TSharedPtr SSEBufferLock; + + // Line-level byte buffer for SSE parsing; fed from SSEStreamBufferPtr. + // UTF-8 multibyte sequences are never split here because lines are extracted whole. + TArray SSERawBuffer; + int32 SSEByteOffset = 0; + // Prevents processing finish_reason more than once if a provider sends duplicate done chunks + bool bResponseComplete = false; + + // Tool call accumulation during streaming; keyed by tool_calls[index] + TMap ToolCallNameMap; + TMap ToolCallArgsMap; + TMap ToolCallIdMap; + + void SendChatCompletionRequest(FAIMessage CurrentMessage); + + void OnResponseProgress(FHttpRequestPtr Request, uint64 BytesSent, uint64 BytesReceived); + void OnRequestComplete(FHttpRequestPtr Request, FHttpResponsePtr Response, bool bSuccess); + + void ParseSSELine(const FString& Line); + void HandleToolCallsDone(); + + // Builds the messages array from system instructions + history + current message appended last + TArray> BuildMessagesArray(FAIMessage CurrentMessage); + + // Builds the tools array from GetAvailableCommands() + TArray> BuildToolsArray(); + + void ResetSSEState(); +}; diff --git a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/OpenRouter/OpenRouterConfig.cpp b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/OpenRouter/OpenRouterConfig.cpp new file mode 100644 index 0000000..7527c84 --- /dev/null +++ b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/OpenRouter/OpenRouterConfig.cpp @@ -0,0 +1,7 @@ +#include "OpenRouter/OpenRouterConfig.h" +#include "OpenRouter/AvatarCoreAIOpenRouter.h" + +UOpenRouterConfig::UOpenRouterConfig(const FObjectInitializer& ObjectInitializer) +{ + AIManagerClass = UAvatarCoreAIOpenRouter::StaticClass(); +} diff --git a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/OpenRouter/OpenRouterConfig.h b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/OpenRouter/OpenRouterConfig.h new file mode 100644 index 0000000..ebad8c1 --- /dev/null +++ b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/OpenRouter/OpenRouterConfig.h @@ -0,0 +1,47 @@ +// Fill out your copyright notice in the Description page of Project Settings. + +#pragma once + +#include "CoreMinimal.h" +#include "AIBaseConfig.h" +#include "OpenRouterConfig.generated.h" + +USTRUCT(BlueprintType) +struct FOpenRouterAISettings +{ + GENERATED_BODY() + + // OpenRouter (or compatible) API base URL + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Settings", meta = (ExposeOnSpawn = "true")) + FString BaseURL = TEXT("https://openrouter.ai/api/v1"); + + //Base URL - Change this to the correct Azure API URL + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Settings", meta = (ExposeOnSpawn = "true")) + FBaseAISettings BaseAISettings; + + // Enable OpenAI-compatible function/tool calling. Disable for models that use a + // non-standard tool format (e.g. some Mistral variants via OpenRouter). + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Settings", meta = (ExposeOnSpawn = "true")) + bool bSendTools = true; + + // Optional: sent as HTTP-Referer header (appears in OpenRouter dashboard analytics) + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Settings", meta = (ExposeOnSpawn = "true")) + FString SiteURL; + + // Optional: sent as X-Title header + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Settings", meta = (ExposeOnSpawn = "true")) + FString AppName; +}; + +UCLASS(Blueprintable, BlueprintType) +class AVATARCORE_AI_API UOpenRouterConfig : public UAIBaseConfig +{ + GENERATED_BODY() + +public: + + UOpenRouterConfig(const FObjectInitializer& ObjectInitializer); + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Settings", meta = (ExposeOnSpawn = "true")) + FOpenRouterAISettings OpenRouterSettings; +}; diff --git a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/RealtimeAPI/AvatarCoreAIRealtime.h b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/RealtimeAPI/AvatarCoreAIRealtime.h index e421c38..d4d6de1 100644 --- a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/RealtimeAPI/AvatarCoreAIRealtime.h +++ b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/RealtimeAPI/AvatarCoreAIRealtime.h @@ -149,7 +149,7 @@ public: void ActivateAI() override; void DeactivateAI() override; void UpdateSession() override; - void SendResponseChild(FAIMessage Message, bool NotifyDelay, bool TriggerResponse) override; + void SendResponseChild(FAIMessage Message, bool NotifyDelay) override; void ClearAI() override; void ConnectToWebSocket(); @@ -162,11 +162,12 @@ public: void WebSocketSendType(const FString& type); UFUNCTION(BlueprintCallable, Category = "AvatarCore AI|RealtimeAPI") - void CreateConversationItem(FAIMessage Message, bool triggerResponse = true); + void CreateConversationItem(FAIMessage Message); UFUNCTION(BlueprintCallable, Category = "AvatarCore AI|RealtimeAPI") void CreateReseponse(); + UFUNCTION(BlueprintCallable, Category = "AvatarCore AI|RealtimeAPI") FString GetCurrentRequestID(); diff --git a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/RealtimeAPI/RealtimeAPIConfig.h b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/RealtimeAPI/RealtimeAPIConfig.h index 0fbcead..29113f9 100644 --- a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/RealtimeAPI/RealtimeAPIConfig.h +++ b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/RealtimeAPI/RealtimeAPIConfig.h @@ -21,17 +21,14 @@ enum class EOpenAIRealtimeVoice : uint8 { verse UMETA(DisplayName = "verse") }; -/** - * - */ -UCLASS() -class AVATARCORE_AI_API URealtimeAPIConfig : public UAIBaseConfig +USTRUCT(BlueprintType) +struct FRealtimeAISettings { GENERATED_BODY() -public: - - URealtimeAPIConfig(const FObjectInitializer& ObjectInitializer); + //Base URL - Change this to the correct Azure API URL + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Settings", meta = (ExposeOnSpawn = "true")) + FBaseAISettings BaseAISettings; //Base URL - Change this to the correct Azure API URL UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Settings", meta = (ExposeOnSpawn = "true")) @@ -41,27 +38,27 @@ public: UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Settings", meta = (ExposeOnSpawn = "true")) bool IsAzureOpenAI = false; - //The OpenAI API Key - UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Settings", meta = (ExposeOnSpawn = "true")) - FString APIKey; - - //OpenAI Model - UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Settings", meta = (ExposeOnSpawn = "true")) - FString Model = "gpt-realtime"; - //OpenAI RealtimeAPI Voice UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Settings", meta = (ExposeOnSpawn = "true")) EOpenAIRealtimeVoice Voice = EOpenAIRealtimeVoice::alloy; - //Max Token per Request + //Shall we forward all audio chunks directly to OpenAI - Does not work well, if we do not forward silence as well UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Settings", meta = (ExposeOnSpawn = "true")) - int MaxTokens = 1500; + bool InputAudioStreaming = false; +}; - //Temperature of the AI model - UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Settings", meta = (ExposeOnSpawn = "true")) - float Temperature = 0.8f; +/** + * + */ +UCLASS() +class AVATARCORE_AI_API URealtimeAPIConfig : public UAIBaseConfig +{ + GENERATED_BODY() + +public: + + URealtimeAPIConfig(const FObjectInitializer& ObjectInitializer); - //Shall we forward all audio chunks directly to OpenAI - Does not work well, if we do not forward silence as well UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Settings", meta = (ExposeOnSpawn = "true")) - bool InputAudioStreaming = false; + FRealtimeAISettings RealtimeSettings; }; diff --git a/Unreal/Plugins/AvatarCore_AI/Source/ThirdParty/MCPServer/FastMCP/.gitignore b/Unreal/Plugins/AvatarCore_AI/Source/ThirdParty/MCPServer/FastMCP/.gitignore deleted file mode 100644 index 9372c7b..0000000 --- a/Unreal/Plugins/AvatarCore_AI/Source/ThirdParty/MCPServer/FastMCP/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/__pycache__/* \ No newline at end of file diff --git a/Unreal/Plugins/AvatarCore_AI/Source/ThirdParty/MCPServer/FastMCP/AddDocumentsToDatabase.bat b/Unreal/Plugins/AvatarCore_AI/Source/ThirdParty/MCPServer/FastMCP/AddDocumentsToDatabase.bat deleted file mode 100644 index f19f42c..0000000 --- a/Unreal/Plugins/AvatarCore_AI/Source/ThirdParty/MCPServer/FastMCP/AddDocumentsToDatabase.bat +++ /dev/null @@ -1,5 +0,0 @@ -@echo off -call %localappdata%/AvatarCore/FastMCPVenv/Scripts/Activate.bat -cd /d "%~dp0" -python AddDocumentsToDatabase.py -cmd \ No newline at end of file diff --git a/Unreal/Plugins/AvatarCore_AI/Source/ThirdParty/MCPServer/FastMCP/AddDocumentsToDatabase.py b/Unreal/Plugins/AvatarCore_AI/Source/ThirdParty/MCPServer/FastMCP/AddDocumentsToDatabase.py deleted file mode 100644 index d92354d..0000000 --- a/Unreal/Plugins/AvatarCore_AI/Source/ThirdParty/MCPServer/FastMCP/AddDocumentsToDatabase.py +++ /dev/null @@ -1,68 +0,0 @@ -# setup_db.py -import os -import re -from document_vectordb import DocumentVectorDB -from colorama import Fore - -def _sanitize_table_name(name: str) -> str: - name = name.strip().lower() - name = re.sub(r'[^a-z0-9]+', '_', name) - name = name.strip('_') - if not name: - return "documents" - return name - - -def setup_database(): - print(Fore.GREEN + "Initializing database...") - db = DocumentVectorDB() - db.create_table() - - # Add sample documents (modify paths as needed) - document_add_folder = os.path.dirname(__file__) + "/documents_to_add" - # Add sample documents (modify paths as needed) - document_added_folder = os.path.dirname(__file__) + "/documents_added" - - if not os.path.exists(document_add_folder): - os.makedirs(document_add_folder) - print(Fore.GREEN + f"Created {document_add_folder} directory. Please add your PDF/text files there.") - return - if not os.path.exists(document_added_folder): - os.makedirs(document_added_folder) - - files_added = 0 - for root, dirs, files in os.walk(document_add_folder): - rel_root = os.path.relpath(root, document_add_folder) - if rel_root == ".": - current_table = "documents" - else: - first_folder = rel_root.split(os.sep)[0] - current_table = _sanitize_table_name(first_folder) - for filename in files: - if filename.endswith(('.pdf', '.txt')): - file_path = os.path.join(root, filename) - if rel_root == ".": - target_root = document_added_folder - else: - target_root = os.path.join(document_added_folder, rel_root) - if not os.path.exists(target_root): - os.makedirs(target_root) - copy_file_path = os.path.join(target_root, filename) - print(Fore.GREEN +f"Adding {file_path} to table {current_table}...") - try: - db.add_document(file_path, table_name=current_table) - files_added += 1 - os.rename(file_path, copy_file_path) - except Exception as e: - print(Fore.RED + f"Error adding {file_path}: {e}") - - db.finalize_db() - print(Fore.GREEN + f"Database setup complete! Added {files_added} documents.") - - # Show stats - stats = db.get_stats() - print(Fore.GREEN + f"Database stats: {stats}") - - -if __name__ == "__main__": - setup_database() \ No newline at end of file diff --git a/Unreal/Plugins/AvatarCore_AI/Source/ThirdParty/MCPServer/FastMCP/FastMCPServer.bat b/Unreal/Plugins/AvatarCore_AI/Source/ThirdParty/MCPServer/FastMCP/FastMCPServer.bat deleted file mode 100644 index f6bef37..0000000 --- a/Unreal/Plugins/AvatarCore_AI/Source/ThirdParty/MCPServer/FastMCP/FastMCPServer.bat +++ /dev/null @@ -1,142 +0,0 @@ -@echo off -setlocal enabledelayedexpansion -title "FastMCP" - -REM ====== Config ====== -set "PY_DOWNLOAD_URL=https://www.python.org/ftp/python/3.10.11/python-3.10.11-amd64.exe" -set "CUDA_DOWNLOAD_URL=https://developer.download.nvidia.com/compute/cuda/12.8.0/local_installers/cuda_12.8.0_571.96_windows.exe" - -REM ====== Config ====== -set "PY_DOWNLOAD_URL=https://www.python.org/ftp/python/3.10.11/python-3.10.11-amd64.exe" -:: Set your required Python version here -set REQUIRED_MAJOR=3 -set REQUIRED_MINOR=10 - -set "VENV_DIR=%LOCALAPPDATA%/AvatarCore/FastMCPVenv" -set "VENV_PY=%VENV_DIR%\Scripts\python.exe" -set "REQ_FILE=%~dp0requirements.txt" -set "TARGET_SCRIPT=%~dp0FastMCPServer.py" - -REM Work from this script�s directory -cd /d "%~dp0" - -setlocal EnableExtensions - -:checkcuda -echo === Checking for CUDA 12.8 === - -REM 1) Is nvcc in PATH? -where nvcc >nul -if errorlevel 1 goto installcuda - -nvcc --version | findstr /i /r /c:"release *12\.8" >nul -if not errorlevel 1 ( - REM Optional: show the version line cleanly - for /f "delims=" %%L in ('nvcc --version ^| findstr /i /c:"release"') do set "NVCC_LINE=%%L" - echo Found: %NVCC_LINE% - echo CUDA 12.8 detected. All good. - goto :checkpython -) - -:installcuda -echo ERROR: - Install CUDA 12.8 first. -start "" "%CUDA_DOWNLOAD_URL%" -pause -exit /b 0 - -:checkpython -echo === Checking for Python %REQUIRED_PY% === -set "PY_PATH=%1" -IF [%1] == [] set "PY_PATH=python" - -:: Check if python command exists -%PY_PATH% --version >nul 2>&1 -if %errorlevel% neq 0 ( - echo ERROR: Python is not installed or not in PATH - start "" "%PY_DOWNLOAD_URL%" - pause - exit /b 1 -) - -:: Get Python version -for /f "tokens=2" %%i in ('%PY_PATH% --version 2^>^&1') do set PYTHON_VERSION=%%i - -echo Found Python version: %PYTHON_VERSION% - -:: Parse version numbers -for /f "tokens=1,2 delims=." %%a in ("%PYTHON_VERSION%") do ( - set CURRENT_MAJOR=%%a - set CURRENT_MINOR=%%b -) - -:: Version comparison logic -set VERSION_OK=0 -if %CURRENT_MAJOR% EQU %REQUIRED_MAJOR% ( - if %CURRENT_MINOR% EQU %REQUIRED_MINOR% ( - set VERSION_OK=1 - ) -) - -:: Display result -if %VERSION_OK% equ 1 ( - echo SUCCESS: Python version is compatible %REQUIRED_MAJOR%.%REQUIRED_MINOR%! - goto :PythonReady -) else ( - echo ERROR: Python version does not match! - start "" "%PY_DOWNLOAD_URL%" - pause - exit /b 1 -) - - -:PythonReady -echo Using Python: %PY_PATH% - -REM ====== Virtual environment ====== -if exist "%VENV_PY%" ( - echo Found existing venv: "%VENV_DIR%" -) else ( - echo ERROR: No venv found. Creating venv at "%VENV_DIR%" ... 1>&2 - %PY_PATH% -m venv "%VENV_DIR%" - if %errorlevel% neq 0 ( - echo Failed to create virtual environment. - pause - exit /b 1 - ) - set "FIRST_SETUP=1" -) - -REM Always upgrade pip once in venv -echo Upgrading pip in venv... 1>&2 -"%VENV_PY%" -m pip install --upgrade pip - -REM Install requirements only on first setup (or if requirements.txt exists and user wants a refresh) -if exist "%REQ_FILE%" ( - if defined FIRST_SETUP ( - echo Installing requirements from "%REQ_FILE%" ... 1>&2 - ) - "%VENV_PY%" -m pip install -r "%REQ_FILE%" - if %errorlevel% neq 0 ( - echo Pip install failed. Check your "requirements.txt". 1>&2 - pause - exit /b 1 - ) - ) else ( - echo No requirements.txt found. Skipping dependency install. - ) - -REM ====== Run the target script ====== -if not exist "%TARGET_SCRIPT%" ( - echo ERROR: "%TARGET_SCRIPT%" not found. 1>&2 - echo Make sure %TARGET_SCRIPT% is next to this script, or update TARGET_SCRIPT path. - pause - exit /b 1 -) - -echo Running %TARGET_SCRIPT% ... -start /B /WAIT "" "%VENV_PY%" "%TARGET_SCRIPT%" %* -set "RUN_EXIT=%ERRORLEVEL%" -echo. -echo %TARGET_SCRIPT% exited with code %RUN_EXIT%. -pause -exit /b %RUN_EXIT% \ No newline at end of file diff --git a/Unreal/Plugins/AvatarCore_AI/Source/ThirdParty/MCPServer/FastMCP/FastMCPServer.py b/Unreal/Plugins/AvatarCore_AI/Source/ThirdParty/MCPServer/FastMCP/FastMCPServer.py deleted file mode 100644 index 2f5ae9a..0000000 --- a/Unreal/Plugins/AvatarCore_AI/Source/ThirdParty/MCPServer/FastMCP/FastMCPServer.py +++ /dev/null @@ -1,41 +0,0 @@ -from fastmcp import FastMCP -from typing import List, Dict, Any -from document_vectordb import DocumentVectorDB -import json -import re - - -mcp = FastMCP("AvatarCoreMCP_1_0", stateless_http=True) - -# Initialize your vector database -db = DocumentVectorDB() -db.create_table() - -@mcp.tool() -def search_information(query: str) -> List[Dict[str, Any]]: - """If you need more information about search this database.""" - try: - query = re.sub('Green Hydrogen Hub Stuttgart', '', query) - query = re.sub('Green Hydrogen Hub', '', query) - query = re.sub('GHH', '', query) - print(query) - results = db.search(query, 3,"documents", True, 48, 12) #Boolean for Cuda based ReRanking - return results - except Exception as e: - return [{"error": f"Search failed: {str(e)}"}] - -@mcp.tool() -def get_database_stats() -> Dict[str, Any]: - """Get statistics about the document database""" - try: - table = db.table - count = table.count_rows() - return { - "total_entries": count, - "table_name": db.table_name - } - except Exception as e: - return {"error": f"Failed to get stats: {str(e)}"} - -if __name__ == "__main__": - mcp.run(transport="http", host="127.0.0.1", port=8000, path="/mcp") \ No newline at end of file diff --git a/Unreal/Plugins/AvatarCore_AI/Source/ThirdParty/MCPServer/FastMCP/StartPythonVenv.bat b/Unreal/Plugins/AvatarCore_AI/Source/ThirdParty/MCPServer/FastMCP/StartPythonVenv.bat deleted file mode 100644 index 937a191..0000000 --- a/Unreal/Plugins/AvatarCore_AI/Source/ThirdParty/MCPServer/FastMCP/StartPythonVenv.bat +++ /dev/null @@ -1,4 +0,0 @@ -@echo off -call %localappdata%/AvatarCore/FastMCPVenv/Scripts/Activate.bat -cd /d "%~dp0" -cmd \ No newline at end of file diff --git a/Unreal/Plugins/AvatarCore_AI/Source/ThirdParty/MCPServer/FastMCP/TestSearchDatabase.bat b/Unreal/Plugins/AvatarCore_AI/Source/ThirdParty/MCPServer/FastMCP/TestSearchDatabase.bat deleted file mode 100644 index 6170ec3..0000000 --- a/Unreal/Plugins/AvatarCore_AI/Source/ThirdParty/MCPServer/FastMCP/TestSearchDatabase.bat +++ /dev/null @@ -1,5 +0,0 @@ -@echo off -call %localappdata%/AvatarCore/FastMCPVenv/Scripts/Activate.bat -cd /d "%~dp0" -python TestSearchDatabase.py -cmd \ No newline at end of file diff --git a/Unreal/Plugins/AvatarCore_AI/Source/ThirdParty/MCPServer/FastMCP/TestSearchDatabase.py b/Unreal/Plugins/AvatarCore_AI/Source/ThirdParty/MCPServer/FastMCP/TestSearchDatabase.py deleted file mode 100644 index cbab6a8..0000000 --- a/Unreal/Plugins/AvatarCore_AI/Source/ThirdParty/MCPServer/FastMCP/TestSearchDatabase.py +++ /dev/null @@ -1,64 +0,0 @@ -from document_vectordb import DocumentVectorDB -import traceback -import time - -table = "documents" - -def DoSearch(): - global table - print("-------------------------", flush=True) - print(f'Which table to search? (empty for {table})', flush=True) - table = input() or table - print('What do you wanna search?', flush=True) - query = input() - - if query == "": - exit(); - - try: - start_time = time.time() - print("Calling db.search...", flush=True) - results = db.search(query, limit=3, table_name=table, rerank=True, candidates=48, batch_size=12) - print(f"db.search returned list of length: {len(results) if results is not None else 'None'}", flush=True) - except Exception as e: - print("Search raised an exception:", flush=True) - print(e, flush=True) - traceback.print_exc() - results = [] - try: - if results and len(results) > 0: - for i, result in enumerate(results, 1): - print(f" {i} {result}", flush=True) - print("-------------------------", flush=True) - else: - print(" No results found", flush=True) - except Exception as e: - print(f"Error printing results: {e}", flush=True) - - print("Execution took: %s seconds" % (time.time() - start_time)) - - DoSearch(); - -if __name__ == "__main__": - import os - try: - print("Testing Document Search...", flush=True) - - # Initialize database - db = DocumentVectorDB() - db.create_table() - - # Get stats - stats = db.get_stats() - print(f"Database stats: {stats}", flush=True) - - DoSearch() - except Exception as e: - print("Fatal error in main:", flush=True) - print(e, flush=True) - traceback.print_exc() - finally: - try: - input("Press Enter to exit...") - except Exception: - pass \ No newline at end of file diff --git a/Unreal/Plugins/AvatarCore_AI/Source/ThirdParty/MCPServer/FastMCP/WipeDatabase.bat b/Unreal/Plugins/AvatarCore_AI/Source/ThirdParty/MCPServer/FastMCP/WipeDatabase.bat deleted file mode 100644 index 9366f93..0000000 --- a/Unreal/Plugins/AvatarCore_AI/Source/ThirdParty/MCPServer/FastMCP/WipeDatabase.bat +++ /dev/null @@ -1,9 +0,0 @@ -@echo off -cd /d "%~dp0" -robocopy documents_added documents_to_add /MOV /S -rmdir /S /Q lancedb -rmdir /S /Q documents_added -echo ----------------------------------------- -echo Farewall, my old friend! -echo ----------------------------------------- -cmd \ No newline at end of file diff --git a/Unreal/Plugins/AvatarCore_AI/Source/ThirdParty/MCPServer/FastMCP/document_vectordb.py b/Unreal/Plugins/AvatarCore_AI/Source/ThirdParty/MCPServer/FastMCP/document_vectordb.py deleted file mode 100644 index 4d2c699..0000000 --- a/Unreal/Plugins/AvatarCore_AI/Source/ThirdParty/MCPServer/FastMCP/document_vectordb.py +++ /dev/null @@ -1,370 +0,0 @@ -import lancedb -import pandas as pd -from sentence_transformers import SentenceTransformer -from sentence_transformers import CrossEncoder -from pdfminer.high_level import extract_text -from langchain_text_splitters import RecursiveCharacterTextSplitter -import pyarrow as pa -import os -import torch -import time -from colorama import Fore -from typing import List, Dict -import re -from hashlib import sha1 - -class DocumentVectorDB: - def __init__(self, db_path: str = "./lancedb"): - if db_path == "./lancedb": - base_dir = os.path.dirname(os.path.abspath(__file__)) - db_path = os.path.join(base_dir, "lancedb") - self.db = lancedb.connect(db_path) - # Model will be initialized in create_table() based on table vector dimension - - #Debug Variables - self.start_time = None - self.end_time = None - - self.model = None - self.model_dim = None - self.model_name = None - self.table_name = "documents" - self.reranker = None - self._load_reranker() - - def _load_reranker(self, model_name: str = "cross-encoder/ms-marco-MiniLM-L-6-v2"): - if self.reranker is None: - if(torch.cuda.is_available()): - self.reranker = CrossEncoder(model_name, device="cuda") - else: - self.reranker = CrossEncoder(model_name) - - def _load_model_for_dim(self, dim: int): - if self.model is not None and self.model_dim == dim: - return - # Select a common model matching the vector dimension - if dim == 384: - self.model_name = 'all-MiniLM-L6-v2' - elif dim == 768: - self.model_name = 'all-mpnet-base-v2' - else: - # Fallback to a widely used 384-d model - self.model_name = 'all-MiniLM-L6-v2' - dim = 384 - self.model = SentenceTransformer(self.model_name) - self.model_dim = dim - - def _infer_table_vector_dim(self) -> int: - try: - # Prefer schema-based detection; vector is commonly a FixedSizeList in LanceDB - field = self.table.schema.field('vector') - t = field.type - if pa.types.is_fixed_size_list(t): - return t.list_size - except Exception: - pass - # Fallback: inspect a sample row - try: - df_sample = self.table.to_pandas(limit=1) - if df_sample is not None and not df_sample.empty: - vec = df_sample.iloc[0].get('vector', []) - if isinstance(vec, list): - return len(vec) - except Exception: - pass - # Default if undetectable - return 384 - - def _ensure_table(self, table_name: str): - if getattr(self, "table", None) is not None and self.table_name == table_name and self.model is not None: - return - self.table_name = table_name - if table_name not in self.db.table_names(): - # Initialize a default 384-d model for a fresh table - self._load_model_for_dim(384) - emb_dim = self.model.get_sentence_embedding_dimension() - # Create with sample data first to enforce schema with correct vector dim - sample_data = pd.DataFrame([{ - "id": "sample", - "content": "sample content", - "source": "sample.txt", - "vector": [0.0] * emb_dim, - "doc_id": "sample.txt", - "chunk_index": 0 - }]) - self.table = self.db.create_table(table_name, sample_data) - # Delete the sample data - self.table.delete("id = 'sample'") - else: - self.table = self.db.open_table(table_name) - # Infer vector dimension from existing table and load matching model - dim = self._infer_table_vector_dim() - self._load_model_for_dim(dim) - - def create_table(self, table_name: str = "documents"): - # Create table schema - schema = { - "id": str, - "content": str, - "source": str, - "vector": list - } - self._ensure_table(table_name) - - def _create_index_for_current_table(self): - try: - row_count = self.table.count_rows() - except Exception: - row_count = 0 - - if row_count < 100: - return - - if row_count < 1000: - num_partitions = 1 - num_sub_vectors = 8 - elif row_count < 10000: - num_partitions = 8 - num_sub_vectors = 16 - elif row_count < 100000: - num_partitions = 32 - num_sub_vectors = 64 - else: - num_partitions = 90 - num_sub_vectors = 96 - - try: - self.table.create_index( - vector_column_name="vector", - index_type="IVF_PQ", - metric="cosine", - num_partitions=num_partitions, - num_sub_vectors=num_sub_vectors - ) - except Exception: - pass - - def clean_extracted_text(self, text: str) -> str: - """ - Cleans up common PDF extraction artifacts: - 1. Removes line-break hyphens. - 2. Replaces excessive whitespace and newlines. - """ - - text = re.sub(r'([a-z])-(\n\s*)(\n?)', r'\1', text, flags=re.IGNORECASE) - text = re.sub(r'\s+', ' ', text).strip() - - return text - - def extract_text_from_pdf(self, pdf_path: str) -> str: - text = "" - # pdfminer.six is often more tolerant of broken PDF structure - try: - text = self.clean_extracted_text(extract_text(pdf_path)) - return text - except Exception as e: - print(f"pdfminer.six failed on {pdf_path}: {e}") - return "" # Or try pypdf as a fallback here - - def chunk_text(self, text: str, chunk_size: int = 128, overlap: int = 16) -> List[str]: - # Use characters (tokens) for chunking, not just words - splitter = RecursiveCharacterTextSplitter( - chunk_size=512, # Set chunk size to token/char count, not word count - chunk_overlap=50, - length_function=len, # Use character length - separators=["\n\n", "\n", ". ", " ", ""] # Hierarchical splitting - ) - # The splitters are highly optimized and handle the logic efficiently - chunks = splitter.split_text(text) - return chunks - - def add_document(self, file_path: str, doc_type: str = "auto", table_name: str = "documents"): - # Ensure model is initialized (in case add_document is used without create_table()) - self._ensure_table(table_name) - - # Extract text based on file type - if doc_type == "auto": - doc_type = "pdf" if file_path.endswith('.pdf') else "txt" - - if doc_type == "pdf": - text = self.extract_text_from_pdf(file_path) - else: - with open(file_path, 'r', encoding='utf-8') as f: - text = f.read() - - # Chunk the text - chunks = self.chunk_text(text) - - print(Fore.GREEN + f"{len(chunks)} chunks added.") - print(Fore.BLUE + chunks[0]) - - # Create embeddings and store - data_to_add = [] - if chunks: - embeddings = self.model.encode( - chunks, - batch_size=64, - convert_to_numpy=True, - normalize_embeddings=True, - ) - base = os.path.basename(file_path) - for i, (chunk, emb) in enumerate(zip(chunks, embeddings)): - if len(chunk) < 1: - continue - did = sha1(f"{base}|{i}|{len(chunk)}".encode("utf-8")).hexdigest()[:16] - doc_data = { - "id": did, - "content": chunk, - "source": file_path, - "vector": emb.tolist(), - "doc_id": base, - "chunk_index": i, - } - data_to_add.append(doc_data) - - # Add all chunks at once - if data_to_add: - df = pd.DataFrame(data_to_add) - self.table.add(df) - self._create_index_for_current_table() - - def finalize_db(self): - self._create_index_for_current_table() - - def DebugTimeIt(self, TimedLabel=""): - if self.start_time is not None: - self.end_time = time.time() - elapsed_time = self.end_time - self.start_time - print(f"{TimedLabel}: Elapsed time {elapsed_time}") - self.start_time = time.time() - - def search(self, query: str, limit: int = 5, table_name: str = "documents", rerank: bool = False, candidates: int = 100, batch_size: int = 64) -> List[Dict]: - # Ensure model is initialized - self._ensure_table(table_name) - query_embedding = self.model.encode(query, normalize_embeddings=True).tolist() - - if rerank: - print("Reranking...") - raw = ( - self.table - .search(query_embedding) - .nprobes(20) - .refine_factor(50) - .limit(candidates) - .to_pandas() - ) - - if raw is None or raw.empty: - return [] - - pairs = [(query, c) for c in raw["content"].tolist()] - - try: - self._load_reranker() - scores = self.reranker.predict(pairs, batch_size=batch_size) - raw["rerank_score"] = scores - reranked = raw.sort_values("rerank_score", ascending=False).head(limit) - except Exception: - reranked = raw.head(limit) # graceful fallback - - results_list = [] - for _, row in reranked.iterrows(): - results_list.append({ - "content": row.get("content", ""), - "source": row.get("source", ""), - "score": float(row.get("rerank_score", 0.0)), - }) - return results_list - else: - # Perform vector search - print("Vector search...") - try: - results_df = ( - self.table - .search(query_embedding) - #.metric("cosine") - .nprobes(20) - .refine_factor(50) - .limit(limit) - .to_pandas() - ) - except Exception: - print("Error") - results_df = self.table.search(query_embedding).limit(limit).to_pandas() - - if results_df is None or results_df.empty: - return [] - - def pick(row, keys, default_value=""): - for k in keys: - if k in row and pd.notna(row.get(k, None)): - return row.get(k) - return default_value - - content_keys = ["content", "text", "chunk", "page_content", "body"] - source_keys = ["source", "path", "file_path", "document", "filename"] - score_keys = ["_distance", "score", "_similarity"] - - results_list = [] - for _, row in results_df.iterrows(): - content = pick(row, content_keys, "") - source = pick(row, source_keys, "") - score = pick(row, score_keys, 0.0) - # Ensure numeric score - try: - score = float(score) - except Exception: - score = 0.0 - results_list.append({ - "content": content, - "source": source, - "score": score, - }) - - return results_list - - def get_stats(self) -> Dict: - try: - tables_info = [] - table_names = list(self.db.table_names()) - - old_table = getattr(self, "table", None) - old_table_name = getattr(self, "table_name", None) - - for name in table_names: - try: - tbl = self.db.open_table(name) - self.table = tbl - self.table_name = name - count = tbl.count_rows() - vector_dim = self._infer_table_vector_dim() - tables_info.append({ - "table_name": name, - "total_chunks": count, - "vector_dim": vector_dim, - }) - except Exception as inner_e: - tables_info.append({ - "table_name": name, - "error": str(inner_e), - }) - - if old_table is not None: - self.table = old_table - if old_table_name is not None: - self.table_name = old_table_name - - return { - "lancedb_version": lancedb.__version__, - "pyarrow_version": pa.__version__, - "torch_version": torch.__version__, - "tables": tables_info, - } - except Exception as e: - return {"error": str(e)} - - def get_tables(self) -> List[str]: - try: - return list(self.db.table_names()) - except Exception: - return [] \ No newline at end of file diff --git a/Unreal/Plugins/AvatarCore_AI/Source/ThirdParty/MCPServer/FastMCP/requirements.txt b/Unreal/Plugins/AvatarCore_AI/Source/ThirdParty/MCPServer/FastMCP/requirements.txt deleted file mode 100644 index 598871a..0000000 --- a/Unreal/Plugins/AvatarCore_AI/Source/ThirdParty/MCPServer/FastMCP/requirements.txt +++ /dev/null @@ -1,32 +0,0 @@ ---index-url https://pypi.org/simple ---extra-index-url https://download.pytorch.org/whl/cu128 ---trusted-host download.pytorch.org ---prefer-binary - - -fastmcp==2.10.6 -lancedb==0.25.3 -pandas==2.3.1 -sentence-transformers==5.1.0 -pdfminer.six==20251107 -torch==2.9.0+cu128 -transformers==4.55.0 -huggingface-hub==0.34.4 -scikit-learn==1.7.1 -scipy==1.15.3 -tqdm==4.67.1 -numpy==2.2.6 -regex==2025.7.34 -safetensors==0.6.1 -pyarrow==21.0.0 -python-dotenv==1.1.1 -requests==2.32.4 -uvicorn==0.35.0 -starlette==0.47.2 -sse-starlette==2.4.1 -httpx==0.28.1 -httpx-sse==0.4.1 -pydantic==2.11.7 -pydantic-settings==2.10.1 -langchain-text-splitters==1.0.0 -PyYAML==6.0.2 diff --git a/Unreal/Plugins/AvatarCore_AI/Source/ThirdParty/MCPServer/FastMCP_ForContentFolder.zip b/Unreal/Plugins/AvatarCore_AI/Source/ThirdParty/MCPServer/FastMCP_ForContentFolder.zip new file mode 100644 index 0000000000000000000000000000000000000000..9313b7981db16dee4fe404d1160d2208c0d7076f GIT binary patch literal 11106 zcmb7q1yo$wwk@uO1c$;Mf_s4AA-KD{Yw+L}+}+(FxI2X4?g4^ZaF-w-`o4Zm?n~dk zzs5LK)F{TBXRW>WJZrB#WyHZD&_JGF=A!8ufBf-3Z!jQ)Ai}zK_L71!^bXGOAfOP> zKm75gsDJ!bb>1?2?_ow2FCsj-#y2gBdt$wASZ zetN=t{+x`6pf8k=k{A&Qibi$i{bMtr*do{#tiAHRWtkhQ*prJ7ydM9bSq$IAqI6Qjs1=7Cxn;~IjewkyU^iNDSS;t<*qpKHL>i~J`6cYr-T_dOq z=0b!qNc*(Mti`6fC~;*T4I$h&?oOOFZ7wy-fVhv1G;=3N&q1=W_z5~Z1?crlke|ha zen~3PKL=^!^1B#!me@`IjIm5f(t3ddxfQz4GqD(2Jh0m+0@wjxQ>%VnMX8pVkWFS{ zK2;GX(9JL)E8KARFNbx5;Vc{SMj`J%!CCwsyU-0IIh|0Ny!CX!0*=Sj;C$#DU|G6!id8nfj5~GP9gWl zc}1|wb$CnZ8pdPrNG-RXI&XWo+O1ee+ElMGyvo?aegG~XcQg+(V^zJ$=g{bp)bGSu z@7)gpzBhLmoU1Q>xIk;gFz>(hODrv*_1J!ODLc}Cg4YGX6m?PILK{C*2Tt`Mr7a( ztv5a=ymmIe$)qqz~0$LONUef>evtD*}Wp1 z;P<(Qy2MlB3wnxLb{Vft*$^b1jcCymII~Z1@NLo#Sl_JWOwf+%=9Xu&h(N3VWI@~= zj-{Y*ZgX+#oDApkh={zf+7H^-;qYu@tUq_pX<69!%h2Gr>%>NVaL>o15##Z9Iu3cH zyf6&Rj?MDQvxv3IlW=gm4m$-tuU4c5`~hC@CsBm^Q<9#REeP00yI)o-){E6@@TVSW z>lzps8vIsR$;R>i5LQcdRMUS6-7{*A-LI(FzeTmTeg^(6EXmh@!S=xvywGb1b?#qO zANMz~+Kz_$_SPQ^^yq%CDt+rN|Enss>e_Y-Y$%?i5^dn0To~rXN>j<(Se%+t*c?A5 zi7seFZ}ptm5TaDcoN2?3Ri>OdwQG8j@*&5j?%@^G3h7p=&^UTFs}QynmIz0u##0Tl zAraWzYfUoc_;h$WTpoAU6~Pg!P+`lERb}PPDM@zX@N?ceX|B6YI%ZhWCyNDEGnG|& zw~f?6(yO8#U2oF$&$AA?nmK}V%O#0iPssW%ChUfv%H1aha&F71fwvf%F2HyUYwc5B z8VnOL0_185sg=W?BWg`a_qRC1Xf^SK0XD@|?_{XWyuvJcd< zBf{K>E%>0rEOS^mx9K(+6O8xP<9cP)m!&`;a+2B-USATaM%c!R%tI%OX>?C@dq=X} zg*?C}`}--dQb}ZEK_}4B_o+xrl-yPIk;U@vep`@ zY9wB0C6X1;au&?$K`yE_PP1#;1rugh(P^7fCqy!>6H3)|8*Qrnn2$W@!)9802OGbC zAw4)d7_)hyKP%OnZJT9aHxz3vxS4AE&4^`-IBPa``(qDDzzn-4&_{rT-5Irci%K23 zkKG24ve^~44)5%GUH}Bev(0mL+Ul{Z(A$*{X2Tj2Q5DTY2lIN%m{_~U7+{U;U7(nNguxe-m1eLJEJq~0jimOn z5R5`HTOYX~OfH)Fw3oASgFyJBaYHU zP)EMaE3BP`QJX@RBy^uE*tSTq_akxdqaH6(N<8x!ys%|SD&O60;%WnwF}^n24G%`5 z#6B>}Tz82}FQ;1*Cl5eZi98E2`GzjM4Q51b>mZn^i9D4h-5;NAr$lUdu5Er3YiXJy zj@w3COO;&(5M_`2z%+;nlJYGd=?X_s@><@z&s$CgG_DFs&6^kzL##83K;yussjh+R z*xZT7t9_EvQcJi?jinDK_48nC>33rL*{xSDtQ$f)rK$yL@W(J-$Tb!~a>p;S=Y*N` zVxUOuG#Dv}`JvQfBhk93AZRp$gr%f{G_WMfYnarQ;9j4mDw*#?#gL=e!Q^eb#3G3d z@r+EnYRqu#MNo@VO(}(g^lCn73cJBAYq2Gcq`--uCq$yRf@Jnih#cB zc_JH93>k!AOJjZKr{+P(t=&@sf2vrnKtP*o7eu>FGig5?uZ;Yf$q{_2gs~@W@AfO{ z&$ut7iz9hqJ+u}*Qc9<2a_}mm5=5o`sX*EhG#8krUcM4n-1)`?ic(kqGBYJ9p)`>z zIQ?zXdLlFNuT~&C3b#2bIj%YIi8XyOG%ESLmex`e zML%G{05PRdL+ypHfd(<|&7|qRVnK8~GzCfkeFIIK?VNxlxCCVxke6w~v68QAJB8ap zD{1iFxiS;L%!#Vi=Hb3KxR3`6BX048sCFs)$an&}$Y$wD(pxeu!7(@6b0A*_X@C&6 zK^f#l6j5-Hy=3ng05O`%vVga;0yByjX5;q3B+ysi?dSCvA%*m%K&RijQ;8sMJ6EM5 zFDuFd1C1|yt0@Mji`u58EVZlV71R;Gfn{A{?`L0S0xfduhs_-7LLvYb2UJ;u-9yr0(g$xBbun)KeUrR>Fam(4By}z)Z~C zx1A5~4z@uAGK5Vob~))Xa)kG*D=ExMfP9U$X!K@nCT?E%2%U%mZ0Z$J-e!RmNKNiQ zA-btqj}F4|nBHam=_j%8A;veT#fQs~4Ry(SKyDG1cv9~6~EOI zt~)o+1haOer4)S+PE*V6$+q_Q^w#T#B~_NulbmaukN8oaUD15IoGqytuFro0713GE zb(F67Qn=XMVV<^sn9P3ALLcw**Xh;H~VQpLGhm7!u`&IzoRfE zb*9^z!ff8=(H(J#PWa?O8MNO-`xt{Xwt<*fE~!lq33f=}fY`&FDCa?eHA!=Rzj&Qt z;Ni5<*GKsHVsgC~UQ;dl*4T$v|Ho&c8z_GgYEBbI8L<09{dr0Ed;3j7oP8GhO}hEI zJRd*S7JBltqe2Z0=)_z&uu<^i8I(GA?*}02UC9FkX2*U8u0*qUE`6R$0Z=AGm!0~3 zd#{8OhhOFjj&o?#ax@&ELdL<=FCOo%-$5%?If0wZqIr`l;GC?Tb$@D~QS1U&|Ki<3 zn2#?k%nn^$(KwCxZA%3+n_?-!q9?x%&^ybdkHqwK$w~;*5OZ8rl7kaK-l{zF%c0jE(VGeAc9}$B}gVgO@Qo`wf__kRy|KX_!q>|!@?>qm4n;D zxP?vTNQ`6CJ*sv4DjoC`qpX}I%z>D3lU@e%eFfcy&bkp{gghv9ZjumBH&@3?=9^>- z-%%BY3GVFXbnH2}K|P>VSsQQ5!v&zpY~pO%|8>FlLqIK5Rta-e75FPFAyic|24oQr z4@Nbn;ra219&3e_F=L$cSQ)z$16|;vkXbIb%bIBb9tK~!&qgQNhN@*V1(Fr`I zlgc4XJrr}T(4xUhtP@T`=87Kj|l6{nc8uD*lx)0PQsTA?xJC_ zg||l|r0KXrJ7t@1BMXTABEF;gf~5Dz92i-b#<2>r?MH{@jL0Cm8jYC)Ofi%ibKb0F z;rodtq1!6Jui;m5MVN)wf2n~A?C;W24?lB9dVnw;UFnTPjW6Q75w%Wjw=dcV@P`0-=6Vj5xlHl*uxueMvi)+EBTpa&|n>G_h^t;ARoh0cz{3|Y{ZCkI_?GidETLJs;Ij@yHkA{s%4EM-NI~# z0=W-$EObzEVa@7QT9S)Y+**}YF#_u9dR%#}?n(enXPNu7pL8qB0G&W}1rLs0%~lO( z`8vA-P@=uroR9Fdb8mlMEx|$mas@-Ep_o4HswB~|nI4eZapD$;hS^Oml_+sFa6Pox zml&BBc?d`nlg6G!80@GKB+ER2Fh85_mJBLQA`ND?2%A~L;;AhhXOPIb6oQ1Ykvzkg zaT_uKbuocc@hd7iRRT;#Iu~C!=vsKf4lsZ2fy28-VKT3?JkwP_`~lpW*pe&EF`%IM z0zFHws-r7UtH-W1q^qAl*rOE$804`T*~B1s0&)D|?05}(m0d0CDMJ3qYKi`*YZ0Uu z0D?yJlc2J(i_!`8X=TINEdT!Q;ZiX-v9*eCJ;CidW*EL!;jNCm^UA;xc`GG#-mwlg z&(C8(#!vm^Wckv)T3zk(w5){)&)_wDtil}@#M>4_!X$rwx3J|CQ$mS1q~J|3&-4dx0{qxzM5#Dq0QCF^A-mZIUO#2 z@{>5(M262%r^O|ha{@4SR#(m<`MyUl& zAq5rdkfN>%cra5b#;LA+zQX{&du_<>!-dYBf_7Xuryq9YnM)+ONpC}%W{8W>wm#z> zVqfCFc}Wc9EnXzls|lrDlai(zAwPfyFH!2 z_^zy!TFcYee}Bcs&CYgfY`+PPg`53@f}>5&koSK6EkjBphz{=-PcNp{6&AVW{)!jD zihlVQJJuj@F2lbSc*v3(fQkI3;wxy3PJCn+yHKKU08Z0nr>l-+ zVzb~B_@j}2YRPITV9;*jRy9rg4*Gd_Pjc2m*!$FN^kDsScmKNv=gWQg|Flcx4L>*< ze)wx(5Q%nU{I!l(L(7&ZooC2W&6<*K~0 zt(DHBZsF42QsRG@;Dw|n6sY0iIvXBh5AY=IOa{{|`&JfuoCxIsAU)yjAtkMB?>+iC zC0!Lret;3-EYZ<$%IvX}fpwtNyxBNA7Duo%X@I`;Lt74r6_c{Ld3|8Qai~eR{k@`j zi+h-}u65y{)P8i9rINC_R++nYTKcE|W~yp+LMtUl32n>eKA@;v9^`%M)ayZF$3C2RAn5H0j#! zM=IMq8yDZf77w!cJ@%eRLFX5`N!vk%x{Z(s3nYrT%n@5jpp?Rr!1-=Ja1uBlZk{$> zoz3mFnWQ5MJO4<9H74Eg=Tf7ght`_K4gcAyYt7WSKm0L&Y2kL!*nJ?djJch#f2Oc4 zdC`7&t?|YKr-E{G-u1wk%+$oqg%ztw^DKhl?IfLrNEdU+@|l3y%$Zhm^=8cx0y#5T zF@iL#ATOg=09s4lKB7{nl~;KVfXVdssQqDxP-q&oz+%}7l4yb{=+yv;h^(q~5@`4>zQmve6&ikgx&T2cHo2b0h}^Lm%Nh6sQoa#K&rO-WqkT zC`1S6s14g@^fE67VO1ItIK)LcDSl?@B#Zjo>s`~R@nv2A#X$~mE-+xbZ+NGO#9e3y z^Y&fq#Q`Ai~ZYy0%(QYX?NDEY-;I~ip zp%k8>rRyPe^|4-2y@lliK!$?&rqQ!pQ#N#=Xk<+F$?Y4FsoM((_?+Q&Yr;w?u0^6F zC^u*e($rTcKXB4hiy~)~zYCf^pOW!NV}p*232utZjmm>AS}YFBNvjDu2NihBR~#&T zp~eZ)DZn|9Kv%r08JkZWN+c?Z5=}aXnOFvHY<}M{h}t-!%hmJ_zl65o+p7(=LR3xl za$qaJ@=ax=Oy+9bNu>1WdoD25aC58ms6{N8CVvMt)sc*l9wKQ$+4@>&ETy)(hgwUq zoE;lt;whG(A?iAgpA;3#mr%^p>Ecp&KaF@^KX4~S#2;FN-=zqC)!m`fnD6r!6<4=X zLp!eEaD1h`E&|uXFrg-~XQE4U^Z@$&DEg3vqD%?}0^)@9@=*lyPV$%D?aM~e7tzQBO|(8Bl7o#j!8pcQZL8|T$~TgB#&Ak1}{hn6+BEu>CkIMJxC zbi3>ny-sSbtYT@g_g#AQtxr9KfDob%)P1dfOPi5d=xte#gTMrA{%pLi2oku!tqT1S*h%}<^ zMZS9zEg0oZ@vedIXfxF$$>>^bh;S|#x%P4+DL`RyaF~igf@Y!6oya8}4NWYeOmR#qnq6F%bdYrh zTn81pcAVrS?s0jKs8W2D2`X_Ujhld{>9y&Nkz{DaykdVO?FWcD#>9A+@B~d=r=Y9L zrINf8W45zjon^^-U zq9;g0~u-d&Y_H!Jp^x zI6&e=4-5nZ2>IX6;}3?m4yGRr|2V^?vv>aO$?eN|Y$-Qrwax%&0Uq<7w?*o`v5lr% z>Hc2NSN^sb#S*@jOffs<`bfN(pMY;QrY~s~U`mu)Z-Z{0d_?{w@$t~nHbm+Z%VCxJ z8o+@JLW6NvJCjZ04!0)GwmWi2!6d=-dy-R_YobM~c@G1PuuaR~t5n~+8JC%0b>U$zxRoCS8?k|g=H*K-6L-cZ1+HJPEz zm?Kv*>7FoK+w}x$t&?dw8V4H5cDh9i!@*9JP@6NM#ka42fKrMha%XfgQ&@54*%0Aw zQU$T4`e3plpO#7JsrbyIGkKKe8M^MJFngy(Eu)a;NHG(FB_LAF%Tf7WE0N=ypG1`# zE^m((5SpyIJ6CB<`IbXOMD+KW}(@?|Qv_ z_5NdYfz$Iyzdc3p_5%e0Nq!pbzdx-9zBp}?x7Yn(FXLiwVr`{lXyy1%<#Vk2x(Z$C3ra+ zDXTMcRhAhWNl^=l73rxeAx!v!eLRAFqRlkLyuDE(Q&RqAIGC)@TU2{GdDD8j{yqf! z7a=GZ+S$t+>VD8S`TsH-40S%44FF!N%`04JS<21t6FD4#0JI~){a>o`+OW(-qv!pUY6#mYD)tcBScz{!WK2#nK?ugDx}jANrg{h`&lS>xl<{BXYL0-Ee% zfB`dC`(?@vd0sRt8k4md8u`6}0}GKRkEdemP3F@4F8|a4i#8uMv$ojaE42-InL$nigRYH>6(nP3 z(h#Pt6IWle1=ku%6-N4IR4}SzkvV^))Tz(v-qK>0&~V=v4CUZiTx!c3-Th?CS65vJ zi-SYZ(dJu*t$8vch>Q~imK20zdRrZLPD8TRSlRo}8X$?$OTT`x)UT40Sme@X+>k(P z(SdR}T{8q=Gv9Dd-(iv~k4i!`$KsHO5 z>m4CSsFU}~$dStxw;7LTqk*x)o}Lbp0il5dbZ=(6t07+^@z~Lzf+IlHJjht;YB7}amZw;Lr&*YooLUQkSA2kFSoa*3JYEC8`C2*Zy0^M2a zh&5-k3#U8#+DtV(vQNR1*YAG=}*m#VggubVQJkJ~K<8EQH^ewoe7`gYi#}biOCNwz0 z1b||z?9#E^!!vt{xXv)?V%ALR#;bYqeW$k*p6@Q4(8|H=-wG&HZeBm)(l=(FaFuTA z8fxw?(Gy(@-52Rn0jPiWR`G(U&l6dP(>V?ijO;4^e4=@t3x9_5oYMbmBKa##kNl!f_0yj; z|7aPx$9Cem0f7?4IHNlB3Jt!2d^!$rHn~ z#pET!v(513rU?55!=pC^`d{h!btCjU@81*rzfY#;oz=@)73&3jop%-R@8Ewww$JtH z%bR~je_0>>ivAJ*FZ6$`exL6B-_f5_^RjdSzPR^CZv(==(Epo?>o>{yBlj<=2mD_M zD2V#!ABJ~eVE_OC literal 0 HcmV?d00001 diff --git a/Unreal/Plugins/AvatarCore_Manager/Content/AvatarCoreManager.uasset b/Unreal/Plugins/AvatarCore_Manager/Content/AvatarCoreManager.uasset index d8366d7..5da1910 100644 --- a/Unreal/Plugins/AvatarCore_Manager/Content/AvatarCoreManager.uasset +++ b/Unreal/Plugins/AvatarCore_Manager/Content/AvatarCoreManager.uasset @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:186bdc2c29df6cb3a815a69d2736a017957135fea6d8405aa61c884c59953f49 -size 1893516 +oid sha256:9379803e1aa60bc5952e611cebb9947689372c8bf255cdc42166c79cb7dc34a7 +size 2129983 diff --git a/Unreal/Plugins/AvatarCore_Manager/Content/StateManagement/States/BP_Configurable_QnA_State.uasset b/Unreal/Plugins/AvatarCore_Manager/Content/StateManagement/States/BP_Configurable_QnA_State.uasset new file mode 100644 index 0000000..0590837 --- /dev/null +++ b/Unreal/Plugins/AvatarCore_Manager/Content/StateManagement/States/BP_Configurable_QnA_State.uasset @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ad7dd381a1621e017d69b112e242a1caba540ceda4cd79b04983c25df6bb238f +size 47348 diff --git a/Unreal/Plugins/AvatarCore_Manager/Content/Widgets/Debug/Modules/W_AvatarCoreModuleEntry.uasset b/Unreal/Plugins/AvatarCore_Manager/Content/Widgets/Debug/Modules/W_AvatarCoreModuleEntry.uasset index 30818e8..b2d52db 100644 --- a/Unreal/Plugins/AvatarCore_Manager/Content/Widgets/Debug/Modules/W_AvatarCoreModuleEntry.uasset +++ b/Unreal/Plugins/AvatarCore_Manager/Content/Widgets/Debug/Modules/W_AvatarCoreModuleEntry.uasset @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:a3b0852ef0d33b6723bf048c0f7ed0e57bd42d1da5e504bcebb91806120f9681 -size 272457 +oid sha256:3683cf131467cae35efc00eaa5f1a6a64e29af3c23c8c21db5602f131a4f6a81 +size 283728 diff --git a/Unreal/Plugins/AvatarCore_Manager/Content/Widgets/Debug/Pages/W_DebugAvatarCoreSTT.uasset b/Unreal/Plugins/AvatarCore_Manager/Content/Widgets/Debug/Pages/W_DebugAvatarCoreSTT.uasset index 7f5768e..ad36bec 100644 --- a/Unreal/Plugins/AvatarCore_Manager/Content/Widgets/Debug/Pages/W_DebugAvatarCoreSTT.uasset +++ b/Unreal/Plugins/AvatarCore_Manager/Content/Widgets/Debug/Pages/W_DebugAvatarCoreSTT.uasset @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:a003ec47034fc18212be19ba614cd62895b94e53cb9a53776a686457ce290d7c -size 402976 +oid sha256:772038ed7c65b69d974467ada2114271d5649786cd18a5e6fec92f3591e444eb +size 419134 diff --git a/Unreal/Plugins/AvatarCore_Manager/Content/Widgets/Debug/Pages/W_DebugAvatarCoreTTS.uasset b/Unreal/Plugins/AvatarCore_Manager/Content/Widgets/Debug/Pages/W_DebugAvatarCoreTTS.uasset index 3f1ce5c..6aacc19 100644 --- a/Unreal/Plugins/AvatarCore_Manager/Content/Widgets/Debug/Pages/W_DebugAvatarCoreTTS.uasset +++ b/Unreal/Plugins/AvatarCore_Manager/Content/Widgets/Debug/Pages/W_DebugAvatarCoreTTS.uasset @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:d1ef777d7b8bfefaefe0ebcffeb3eb14b807802ea715ccaab9e42ac1cda10431 -size 268822 +oid sha256:ecda1f8d3ffcb03a527c230c95197944597b8b887b0871179e2341b27dd47a12 +size 265549 diff --git a/Unreal/Plugins/AvatarCore_Manager/Content/Widgets/StartupScreen/W_AvatarCoreStartupScreen.uasset b/Unreal/Plugins/AvatarCore_Manager/Content/Widgets/StartupScreen/W_AvatarCoreStartupScreen.uasset index d124f2c..a2a995d 100644 --- a/Unreal/Plugins/AvatarCore_Manager/Content/Widgets/StartupScreen/W_AvatarCoreStartupScreen.uasset +++ b/Unreal/Plugins/AvatarCore_Manager/Content/Widgets/StartupScreen/W_AvatarCoreStartupScreen.uasset @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:a27f87304c545dd5173df8aac4044d93ffece686fd40a8499cdbfe6199bfd2de -size 77758 +oid sha256:c6828a85b493eff2d015b466dbbdb6d6d084cc1c638a7d62fe9ccbbbb82350a3 +size 80411 diff --git a/Unreal/Plugins/AvatarCore_Manager/Source/AvatarCore_Manager/Private/FL_AvatarCoreManager.cpp b/Unreal/Plugins/AvatarCore_Manager/Source/AvatarCore_Manager/Private/FL_AvatarCoreManager.cpp index 7242df3..f90c5ce 100644 --- a/Unreal/Plugins/AvatarCore_Manager/Source/AvatarCore_Manager/Private/FL_AvatarCoreManager.cpp +++ b/Unreal/Plugins/AvatarCore_Manager/Source/AvatarCore_Manager/Private/FL_AvatarCoreManager.cpp @@ -17,3 +17,144 @@ void UFL_AvatarCoreManager::BindTTSToAIManager(UTTSManagerBase* TTSManager, UAIB AIManager->OnAudioChunk.AddDynamic(TTSManager, &UTTSManagerBase::AddAudioChunk); } } + +ETTSLanguage UFL_AvatarCoreManager::ConvertLanguageToTTS(ELanguage Language) +{ + switch (Language) + { + case ELanguage::ar: + return ETTSLanguage::ar; + + case ELanguage::de: + return ETTSLanguage::de; + + case ELanguage::en: + return ETTSLanguage::en; + + case ELanguage::fr: + return ETTSLanguage::fr; + + case ELanguage::el: + return ETTSLanguage::el; + + case ELanguage::hi: + return ETTSLanguage::hi; + + case ELanguage::it: + return ETTSLanguage::it; + + case ELanguage::ja: + return ETTSLanguage::ja; + + case ELanguage::ko: + return ETTSLanguage::ko; + + case ELanguage::zh: + return ETTSLanguage::zh; + + case ELanguage::nl: + return ETTSLanguage::nl; + + case ELanguage::pl: + return ETTSLanguage::pl; + + case ELanguage::pt: + return ETTSLanguage::pt; + + case ELanguage::ro: + return ETTSLanguage::ro; + + case ELanguage::ru: + return ETTSLanguage::ru; + + case ELanguage::es: + return ETTSLanguage::es; + + case ELanguage::cs: + return ETTSLanguage::cs; + + case ELanguage::tr: + return ETTSLanguage::tr; + + case ELanguage::uk: + return ETTSLanguage::uk; + + case ELanguage::hu: + return ETTSLanguage::hu; + + case ELanguage::NONE: + default: + return ETTSLanguage::NONE; + } +} + +ESTTLanguage UFL_AvatarCoreManager::ConvertLanguageToSTT(ELanguage Language) +{ + switch (Language) + { + case ELanguage::ar: + return ESTTLanguage::ar; + + case ELanguage::de: + return ESTTLanguage::de; + + case ELanguage::en: + return ESTTLanguage::en; + + case ELanguage::fr: + return ESTTLanguage::fr; + + case ELanguage::el: + return ESTTLanguage::el; + + case ELanguage::hi: + return ESTTLanguage::hi; + + case ELanguage::it: + return ESTTLanguage::it; + + case ELanguage::ja: + return ESTTLanguage::ja; + + case ELanguage::ko: + return ESTTLanguage::ko; + + case ELanguage::zh: + return ESTTLanguage::zh; + + case ELanguage::nl: + return ESTTLanguage::nl; + + case ELanguage::pl: + return ESTTLanguage::pl; + + case ELanguage::pt: + return ESTTLanguage::pt; + + case ELanguage::ro: + return ESTTLanguage::ro; + + case ELanguage::ru: + return ESTTLanguage::ru; + + case ELanguage::es: + return ESTTLanguage::es; + + case ELanguage::cs: + return ESTTLanguage::cs; + + case ELanguage::tr: + return ESTTLanguage::tr; + + case ELanguage::uk: + return ESTTLanguage::uk; + + case ELanguage::hu: + return ESTTLanguage::hu; + + case ELanguage::NONE: + default: + return ESTTLanguage::NONE; + + } +} diff --git a/Unreal/Plugins/AvatarCore_Manager/Source/AvatarCore_Manager/Public/AvatarCore_ManagerEnums.h b/Unreal/Plugins/AvatarCore_Manager/Source/AvatarCore_Manager/Public/AvatarCore_ManagerEnums.h index 1f3169a..10065ae 100644 --- a/Unreal/Plugins/AvatarCore_Manager/Source/AvatarCore_Manager/Public/AvatarCore_ManagerEnums.h +++ b/Unreal/Plugins/AvatarCore_Manager/Source/AvatarCore_Manager/Public/AvatarCore_ManagerEnums.h @@ -36,25 +36,47 @@ enum class EAvatarState : uint8 { UENUM(BlueprintType) enum class ELanguage : uint8 { - NONE UMETA(DisplayName = "No language is enforced"), - Arabic UMETA(DisplayName = "arabic"), - German UMETA(DisplayName = "german"), - English UMETA(DisplayName = "english"), - French UMETA(DisplayName = "french"), - Greek UMETA(DisplayName = "greek"), - Hindi UMETA(DisplayName = "hindi"), - Italian UMETA(DisplayName = "italian"), - Japanese UMETA(DisplayName = "japanese"), - Korean UMETA(DisplayName = "korean"), - MandarinChinese UMETA(DisplayName = "mandarin chinese"), - Dutch UMETA(DisplayName = "dutch"), - Polish UMETA(DisplayName = "polish"), - Portuguese UMETA(DisplayName = "Portugiesisch"), - Romanian UMETA(DisplayName = "romanian"), - Russian UMETA(DisplayName = "russian"), - Spanish UMETA(DisplayName = "spanish"), - Czech UMETA(DisplayName = "czech"), - Turkish UMETA(DisplayName = "turkish"), - Ukrainian UMETA(DisplayName = "ukrainian"), - Hungarian UMETA(DisplayName = "hungarian") + NONE UMETA(DisplayName = "Unset"), + en UMETA(DisplayName = "English"), + fr UMETA(DisplayName = "French"), + de UMETA(DisplayName = "German"), + es UMETA(DisplayName = "Spanish"), + pt UMETA(DisplayName = "Portuguese"), + zh UMETA(DisplayName = "Chinese"), + ja UMETA(DisplayName = "Japanese"), + hi UMETA(DisplayName = "Hindi"), + it UMETA(DisplayName = "Italian"), + ko UMETA(DisplayName = "Korean"), + nl UMETA(DisplayName = "Dutch"), + pl UMETA(DisplayName = "Polish"), + ru UMETA(DisplayName = "Russian"), + sv UMETA(DisplayName = "Swedish"), + tr UMETA(DisplayName = "Turkish"), + tl UMETA(DisplayName = "Filipino"), + bg UMETA(DisplayName = "Bulgarian"), + ro UMETA(DisplayName = "Romanian"), + ar UMETA(DisplayName = "Arabic"), + cs UMETA(DisplayName = "Czech"), + el UMETA(DisplayName = "Greek"), + fi UMETA(DisplayName = "Finnish"), + hr UMETA(DisplayName = "Croatian"), + ms UMETA(DisplayName = "Malay"), + sk UMETA(DisplayName = "Slovak"), + da UMETA(DisplayName = "Danish"), + ta UMETA(DisplayName = "Tamil"), + uk UMETA(DisplayName = "Ukrainian"), + hu UMETA(DisplayName = "Hungarian"), + no UMETA(DisplayName = "Norwegian"), + vi UMETA(DisplayName = "Vietnamese"), + bn UMETA(DisplayName = "Bengali"), + th UMETA(DisplayName = "Thai"), + he UMETA(DisplayName = "Hebrew"), + ka UMETA(DisplayName = "Georgian"), + id UMETA(DisplayName = "Indonesian"), + te UMETA(DisplayName = "Telugu"), + gu UMETA(DisplayName = "Gujarati"), + kn UMETA(DisplayName = "Kannada"), + ml UMETA(DisplayName = "Malayalam"), + mr UMETA(DisplayName = "Marathi"), + pa UMETA(DisplayName = "Punjabi"), }; \ No newline at end of file diff --git a/Unreal/Plugins/AvatarCore_Manager/Source/AvatarCore_Manager/Public/FL_AvatarCoreManager.h b/Unreal/Plugins/AvatarCore_Manager/Source/AvatarCore_Manager/Public/FL_AvatarCoreManager.h index e29c788..b18b71d 100644 --- a/Unreal/Plugins/AvatarCore_Manager/Source/AvatarCore_Manager/Public/FL_AvatarCoreManager.h +++ b/Unreal/Plugins/AvatarCore_Manager/Source/AvatarCore_Manager/Public/FL_AvatarCoreManager.h @@ -4,6 +4,8 @@ #include "CoreMinimal.h" #include "Kismet/BlueprintFunctionLibrary.h" +#include "AvatarCore_ManagerEnums.h" +#include "STTStructs.h" // Forward declarations class UTTSManagerBase; class UAvatarCoreAIRealtime; @@ -46,4 +48,10 @@ class AVATARCORE_MANAGER_API UFL_AvatarCoreManager : public UBlueprintFunctionLi UFUNCTION(BlueprintCallable, Category = "AvatarCoreManager") static void BindTTSToAIManager(UTTSManagerBase* TTSManager, UAIBaseManager* AIManager); + + UFUNCTION(BlueprintCallable, Category = "AvatarCoreManager") + static ETTSLanguage ConvertLanguageToTTS(ELanguage Language); + + UFUNCTION(BlueprintCallable, Category = "AvatarCoreManager") + static ESTTLanguage ConvertLanguageToSTT(ELanguage Language); }; diff --git a/Unreal/Plugins/AvatarCore_MetaHuman/Content/Animation/AnimBPs/AvatarCore_AnimInst_BodyForRetarget.uasset b/Unreal/Plugins/AvatarCore_MetaHuman/Content/Animation/AnimBPs/AvatarCore_AnimInst_BodyForRetarget.uasset index d82c0d2..6e0f9d8 100644 --- a/Unreal/Plugins/AvatarCore_MetaHuman/Content/Animation/AnimBPs/AvatarCore_AnimInst_BodyForRetarget.uasset +++ b/Unreal/Plugins/AvatarCore_MetaHuman/Content/Animation/AnimBPs/AvatarCore_AnimInst_BodyForRetarget.uasset @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:cf10bb6ce6f516e3a4f0db5f3518b71d70c595328cc6a9ff00248b366d1497cf -size 698476 +oid sha256:64cb7924b0f197d2394889649ea54a8d0a00fa1ef4c8fe58e0b431a8609a695f +size 699831 diff --git a/Unreal/Plugins/AvatarCore_MetaHuman/Content/Animation/AnimBPs/AvatarCore_AnimInst_Face.uasset b/Unreal/Plugins/AvatarCore_MetaHuman/Content/Animation/AnimBPs/AvatarCore_AnimInst_Face.uasset index ab7e3fb..1c46a9e 100644 --- a/Unreal/Plugins/AvatarCore_MetaHuman/Content/Animation/AnimBPs/AvatarCore_AnimInst_Face.uasset +++ b/Unreal/Plugins/AvatarCore_MetaHuman/Content/Animation/AnimBPs/AvatarCore_AnimInst_Face.uasset @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:aded6fec950125c7f1ce866302ffc36603c424e46f1ab7e2a9960bf51ba1f95d -size 2029779 +oid sha256:557ffa00af9b85122b00e2c906dc341d4c8e5285536f5f337513c13476fe8c56 +size 2029223 diff --git a/Unreal/Plugins/AvatarCore_MetaHuman/Content/BP/MetaHuman/BaseAvatar.uasset b/Unreal/Plugins/AvatarCore_MetaHuman/Content/BP/MetaHuman/BaseAvatar.uasset index b2cfb29..5f2152c 100644 --- a/Unreal/Plugins/AvatarCore_MetaHuman/Content/BP/MetaHuman/BaseAvatar.uasset +++ b/Unreal/Plugins/AvatarCore_MetaHuman/Content/BP/MetaHuman/BaseAvatar.uasset @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:3a79b6bf69595889e0d5743c7b398304a01b0344ab1888ff634b105811ae1492 -size 2485593 +oid sha256:22d7cc09dd8c7265ead02841968633ea72ec7c5e7d0ac82f180d9da451760850 +size 2490354 diff --git a/Unreal/Plugins/AvatarCore_MetaHuman/Content/Materials/MI_GrayTexture_Body_Cascadeur.uasset b/Unreal/Plugins/AvatarCore_MetaHuman/Content/Materials/MI_GrayTexture_Body_Cascadeur.uasset new file mode 100644 index 0000000..aafac6e --- /dev/null +++ b/Unreal/Plugins/AvatarCore_MetaHuman/Content/Materials/MI_GrayTexture_Body_Cascadeur.uasset @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b511649c7af162a2e95ef35e3b4b6546aaa4e167609c3161b493c1e6a3f2a3b1 +size 15607 diff --git a/Unreal/Plugins/AvatarCore_MetaHuman/Content/Materials/MI_GrayTexture_Head.uasset b/Unreal/Plugins/AvatarCore_MetaHuman/Content/Materials/MI_GrayTexture_Head.uasset index 09dd9a6..dd1a1f1 100644 --- a/Unreal/Plugins/AvatarCore_MetaHuman/Content/Materials/MI_GrayTexture_Head.uasset +++ b/Unreal/Plugins/AvatarCore_MetaHuman/Content/Materials/MI_GrayTexture_Head.uasset @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:b881c27ad3d10a98ed810be60d1e21ad60f443b3f78042ba0dda354ed71967b6 -size 8159 +oid sha256:3db89f023858f796d5e8bbb7f89078d76f30a982a9bd2a677745f867b18d57f4 +size 10555 diff --git a/Unreal/Plugins/AvatarCore_MetaHuman/Content/Materials/MI_GrayTexture_Head_Cascadeur.uasset b/Unreal/Plugins/AvatarCore_MetaHuman/Content/Materials/MI_GrayTexture_Head_Cascadeur.uasset new file mode 100644 index 0000000..dd8d282 --- /dev/null +++ b/Unreal/Plugins/AvatarCore_MetaHuman/Content/Materials/MI_GrayTexture_Head_Cascadeur.uasset @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4332ab1d2d1e3ec57f5d0ee8052b42b8cb1a5295d29ba4a754ff9b1bddc3f934 +size 13780 diff --git a/Unreal/Plugins/AvatarCore_MetaHuman/Content/Materials/M_GrayTexture_Body.uasset b/Unreal/Plugins/AvatarCore_MetaHuman/Content/Materials/M_GrayTexture_Body.uasset index 58d5462..bbeda0f 100644 --- a/Unreal/Plugins/AvatarCore_MetaHuman/Content/Materials/M_GrayTexture_Body.uasset +++ b/Unreal/Plugins/AvatarCore_MetaHuman/Content/Materials/M_GrayTexture_Body.uasset @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:6467a5166db099d02d523d681e1f196174692bc9026682fcec541babf1627c04 -size 20010 +oid sha256:60a111d26377602e25465f135c1ad927d5b3465b6648eefbb3c8421dcbd2436e +size 28791 diff --git a/Unreal/Plugins/AvatarCore_MetaHuman/Content/Materials/M_GrayTexture_Head.uasset b/Unreal/Plugins/AvatarCore_MetaHuman/Content/Materials/M_GrayTexture_Head.uasset index 0bc74bc..1643389 100644 --- a/Unreal/Plugins/AvatarCore_MetaHuman/Content/Materials/M_GrayTexture_Head.uasset +++ b/Unreal/Plugins/AvatarCore_MetaHuman/Content/Materials/M_GrayTexture_Head.uasset @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:994449ef905205235c458869b111b1c74fcfc787ee8c9319a7c0bf88f6211fb9 -size 18722 +oid sha256:f4299cc5d140e749a860cc07b5aa2c936eb91a04cf938c6afd4eac87e0f965e7 +size 25730 diff --git a/Unreal/Plugins/AvatarCore_STT/CLAUDE.md b/Unreal/Plugins/AvatarCore_STT/CLAUDE.md new file mode 100644 index 0000000..1160c6a --- /dev/null +++ b/Unreal/Plugins/AvatarCore_STT/CLAUDE.md @@ -0,0 +1,147 @@ +# AvatarCore_STT Plugin + +Speech-to-text plugin for Unreal Engine. Audio flows through a linear chain of modules: + +``` +STTRecorder → [STTPreprocessor, ...] → STTProcessor → transcription result +``` + +The chain is assembled in `STTManagerBase::InitSTTManager` using `BindUFunction` with the string name `"OnChunkReceived"`. The UE reflection system resolves the correct virtual override at bind time, so the manager code does not need to change when signatures change. + +--- + +## ESTTChainState — Pipeline Signal + +Every audio chunk carries an `ESTTChainState` (defined in `Public/STTStructs.h`): + +| Value | Meaning | +|-------|---------| +| `Processing` | Normal audio — buffer, process, pass through | +| `Finalizing` | End of utterance — flush buffers and trigger transcription | +| `Discarding` | BLOCKED/abort — clear all buffers, cancel in-flight requests | + +**Rules:** +- Recorders always emit `Processing` — they have no concept of "final". +- PTT preprocessor emits `Finalizing` when the button is released (→ SILENCE) and `Discarding` when the system becomes BLOCKED. +- VAD preprocessor emits `Finalizing` on the last postroll silence chunk, then calls `UserSpeechStateChanged(SILENCE)` for UI purposes only. It emits `Discarding` from `OnUserSpeechStateChanged` when BLOCKED. +- Pass-through preprocessors (Converter, Debugger, SpeexDSP, WebRTC) forward `ChainState` unchanged. They must pass `Finalizing`/`Discarding` through even when `PCMData` is empty. +- Buffer preprocessor reacts to `ChainState` in-band — no `OnSpeechStateChanged` subscription. +- Processors react to `ChainState` only — no `OnSpeechStateChanged` subscriptions. + +**Processors still call `UserSpeechStateChanged`** after transcription completes (for UI state), but they do NOT subscribe to it. + +--- + +## Delegate Types + +```cpp +// STTRecorderBase.h +DECLARE_DELEGATE_ThreeParams(FDelegateUnprocessedChunkReceived, TArray, FAudioInformation, ESTTChainState); + +// STTPreprocessorBase.h +DECLARE_DELEGATE_ThreeParams(FDelegateProcessedChunk, TArray, FAudioInformation, ESTTChainState); +``` + +--- + +## Module Types + +### STTRecorder (`Public/Recorder/STTRecorderBase.h`) + +Produces audio chunks from a source (microphone, file, pixel stream). Fires `OnChunkReceived` delegate with `ESTTChainState::Processing`. Has no knowledge of speech state. + +Implementations: `STTRecorderMicrophone` (PortAudio), `STTRecorderPrimaryMicrophone`, `STTRecorderUnrealMicrophone`, `STTRecorderDebugFile`, `STTRecorderAudioData`. + +### STTPreprocessor (`Public/Preprocessor/STTPreprocessorBase.h`) + +Chained in sequence. Each receives `OnChunkReceived` and fires `OnChunkProcessed` to the next stage. Both delegates carry `ESTTChainState`. + +| Class | Role | +|-------|------| +| `STTPreprocessorConverter` | Stereo→mono, resample to target rate | +| `STTPreprocessorWebRTC` | WebRTC APM (echo cancel, noise suppress, AGC) | +| `STTPreprocessorSpeexDSP` | Speex noise suppression / echo cancel | +| `STTPreprocessorPTT` | Gates audio by PTT button state; emits Finalizing/Discarding | +| `STTPreprocessorVAD` | Voice activity detection; emits Finalizing after postroll, Discarding on BLOCKED | +| `STTPreprocessorBuffer` | Accumulates chunks to a fixed buffer size before forwarding; `bDiscardWhenNotFilledFullyOnce` drops short utterances | +| `STTPreprocessorDebugger` | Writes audio passing through it to a WAV file | + +**STTPreprocessorBuffer — `bDiscardWhenNotFilledFullyOnce`:** +When enabled, if a `Finalizing` signal arrives before the buffer has ever dispatched a full-size `Processing` chunk in the current utterance, it sends `Discarding` instead. This silently drops very short accidental utterances without sending them to the transcription service. + +### STTProcessor (`Public/Processor/STTProcessorBase.h`) + +Receives the final audio. On `Finalizing`: trigger transcription. On `Discarding`: cancel/clear everything. + +| Class | Backend | +|-------|---------| +| `STTProcessorAzure` | Microsoft Azure Cognitive Services (streaming, continuous) | +| `STTProcessorWhisper` | OpenAI Whisper / GPT-4o Transcribe (batch HTTP) | +| `STTParakeetProcessorBase` | Local NVIDIA NeMo Parakeet via TCP (JSON protocol) | +| `STTProcessorRealtimeAPI` | OpenAI Realtime API (forwards audio directly) | +| `STTProcessorDebugSaveWav` | Saves all received audio to a WAV file | + +--- + +## Configuration + +All modules are configured via `USTTBaseProcessorConfig` (a UObject subclass per processor type). Base settings are in `FSTTBaseSettings` (`Public/STTStructs.h`): + +- `bUsePTT` — Push-to-talk vs. freespeech (VAD) mode +- `bCanInterrupt` — Whether user speech can interrupt the avatar +- `FreespeechPostRollTime` — Seconds of silence after speech before `Finalizing` is emitted +- `PTTPostRollTime` — Seconds after PTT release before `Finalizing` (currently unused — PTT emits Finalizing immediately on release) +- `MaxTalkingTime` — Hard timeout on PTT press duration +- `VADSettings` — Mode, min speech time, min amplitude threshold, speech-while-blocked threshold +- `WebRTCSettings` — Echo cancellation, noise suppression, AGC flags +- `SpeexDSPSettings` — Speex processing entries +- `STTReplacements` — Word replacement pairs applied to final transcription +- `STTSpecialWords` — Hints passed to transcription service for uncommon words + +--- + +## Key Files + +``` +Public/ + STTStructs.h — ESTTChainState, ESTTTalkingState, FAudioInformation, FSTTBaseSettings + STTManagerBase.h/.cpp — Pipeline assembly, state machine, delegate wiring + Recorder/STTRecorderBase.h — FDelegateUnprocessedChunkReceived + Preprocessor/STTPreprocessorBase.h — FDelegateProcessedChunk, virtual OnChunkReceived + Processor/STTProcessorBase.h — virtual OnChunkReceived, OnTranscriptionResult helpers + +Private/ + STTManagerBase.cpp — InitSTTManager (BindUFunction chain), UserSpeechStateChanged + Preprocessor/STTPreprocessorPTT.cpp — Finalizing on SILENCE, Discarding on BLOCKED + Preprocessor/STTPreprocessorVAD.cpp — Finalizing after postroll, Discarding on BLOCKED + Preprocessor/STTPreprocessorBuffer.cpp — ChainState-driven flush, bDiscardWhenNotFilledFullyOnce + Processor/Azure/STTProcessorAzure.cpp — Streaming Azure recognition + Processor/Parakeet/STTParakeetProcessorBase.cpp — TCP JSON protocol to Python server + Processor/Whisper/STTProcessorWhisper.cpp — Batch HTTP to OpenAI +``` + +--- + +## State Machine (ESTTTalkingState) + +Used for UI and for VAD/PTT internal logic only. Processors do NOT subscribe to `OnSpeechStateChanged`. + +``` +SILENCE ──(VAD/PTT detects speech)──▶ TALKING +TALKING ──(VAD postroll / PTT release)──▶ SILENCE [Finalizing propagates through chain] +TALKING ──(SetBlocked)──▶ BLOCKED [Discarding propagates through chain] +BLOCKED ──(SetBlocked false / interrupt)──▶ SILENCE +ANY ──(transcription complete)──▶ SILENCE +``` + +`TRANSCRIBING` is a transitional state set by Whisper before sending an HTTP request; other processors do not use it. + +--- + +## Common Pitfalls + +- **Pass-through preprocessors must forward `Finalizing`/`Discarding` even on empty `PCMData`.** The Converter, SpeexDSP, and WebRTC all have early-return guards for empty/misaligned data — these guards check `ChainState != Processing` before returning so control signals are not swallowed. +- **PTT emits an empty `TArray` with `Finalizing`.** Processors must guard against transcribing zero-length audio (they already do via `BufferedPCMData.Num() == 0` checks). +- **Azure runs a background thread (`FAzureRunnable`).** `StopRecognition(false)` signals a graceful stop; the runnable delivers the final result via `OnRecognized`/`OnRunnableEnded` callbacks on the game thread. `StopRecognition(true)` is a forced abort (used on `Discarding`). +- **Parakeet communicates over TCP with a local Python server** (`ParakeetSTT.bat`). In editor (`bKeepAlive=true`) the Python process is kept alive between PIE sessions to avoid restart overhead. +- **`BindUFunction` matches by string name and delegate parameter types.** All `OnChunkReceived` overrides must have exactly the same signature as the base UFUNCTION or the bind will fail at runtime. diff --git a/Unreal/Plugins/AvatarCore_STT/Content/Preprocessor/STTPreprocessor250ms.uasset b/Unreal/Plugins/AvatarCore_STT/Content/Preprocessor/STTPreprocessor250ms.uasset index 61c4272..926d631 100644 --- a/Unreal/Plugins/AvatarCore_STT/Content/Preprocessor/STTPreprocessor250ms.uasset +++ b/Unreal/Plugins/AvatarCore_STT/Content/Preprocessor/STTPreprocessor250ms.uasset @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:d6fe044af2b2ba66d51c2d14f5f78c2c6ebf595ed7dd3c7d20665c6bfbefbd59 -size 5954 +oid sha256:abf2833a613c21df1c8ceee183d883d8911c150ad20535ad90bc9df293ccbf71 +size 6018 diff --git a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/AvatarCore_STT.Build.cs b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/AvatarCore_STT.Build.cs index 7cc6ed9..c71f1be 100644 --- a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/AvatarCore_STT.Build.cs +++ b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/AvatarCore_STT.Build.cs @@ -39,10 +39,10 @@ public class AvatarCore_STT : ModuleRules } ); - // Ensure ThirdParty/CoquiTTS is packaged in all builds (including shipping) - string CoquiTTSPath = System.IO.Path.Combine(ModuleDirectory, "..", "ThirdParty", "Parakeet"); - RuntimeDependencies.Add(System.IO.Path.Combine(CoquiTTSPath, "*.*")); - RuntimeDependencies.Add(System.IO.Path.Combine(CoquiTTSPath, "**", "*.*")); + // Ensure ThirdParty/Parakeet is packaged in all builds (including shipping) + string ParakeetTTSPath = System.IO.Path.Combine(ModuleDirectory, "..", "ThirdParty", "Parakeet"); + RuntimeDependencies.Add(System.IO.Path.Combine(ParakeetTTSPath, "*.*")); + RuntimeDependencies.Add(System.IO.Path.Combine(ParakeetTTSPath, "**", "*.*")); PublicIncludePaths.Add(Path.Combine(ModuleDirectory, "..", "ThirdParty", "fvad", "include")); PublicAdditionalLibraries.Add(Path.Combine(ModuleDirectory, "..", "ThirdParty", "fvad", "lib", "fvad.lib")); diff --git a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Preprocessor/STTPreprocessorBuffer.cpp b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Preprocessor/STTPreprocessorBuffer.cpp index 7776e45..9ca2e02 100644 --- a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Preprocessor/STTPreprocessorBuffer.cpp +++ b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Preprocessor/STTPreprocessorBuffer.cpp @@ -7,42 +7,62 @@ void USTTPreprocessorBuffer::InitSTTPreprocessor(USTTManagerBase* BaseSTTManager, FSTTBaseSettings InSTTBaseSettings, bool InDebugMode) { USTTPreprocessorBase::InitSTTPreprocessor(BaseSTTManager, InSTTBaseSettings, InDebugMode); - if (FlushOnSilence) - BaseSTTManager->OnSpeechStateChanged.AddUniqueDynamic(this, &USTTPreprocessorBuffer::OnSpeechChanged); } -void USTTPreprocessorBuffer::OnChunkReceived(TArray PCMData, FAudioInformation AudioInformation) +void USTTPreprocessorBuffer::OnChunkReceived(TArray PCMData, FAudioInformation AudioInformation, ESTTChainState ChainState) { + if (ChainState == ESTTChainState::Discarding) + { + Buffer.Empty(); + bHasFilledFully = false; + OnChunkProcessed.ExecuteIfBound({}, AudioInformation, ESTTChainState::Discarding); + return; + } + Buffer.Append(PCMData); BufferAudioInformation = AudioInformation; + + if (ChainState == ESTTChainState::Finalizing) + { + if (bDiscardWhenNotFilledFullyOnce && !bHasFilledFully) + { + Buffer.Empty(); + OnChunkProcessed.ExecuteIfBound({}, BufferAudioInformation, ESTTChainState::Discarding); + } + else + { + if (Buffer.Num() > 0) + OnChunkProcessed.ExecuteIfBound(Buffer, BufferAudioInformation, ESTTChainState::Finalizing); + Buffer.Empty(); + bHasFilledFully = false; + } + return; + } + + const uint32 SampleCount = GetSampleCount(AudioInformation); if (CanOverflow) { - if ((uint32)Buffer.Num() > GetSampleCount(AudioInformation)) { - OnChunkProcessed.ExecuteIfBound(Buffer, AudioInformation); + if ((uint32)Buffer.Num() > SampleCount) { + OnChunkProcessed.ExecuteIfBound(Buffer, AudioInformation, ESTTChainState::Processing); Buffer.Empty(); + bHasFilledFully = true; } } else { - while ((uint32)Buffer.Num() > GetSampleCount(AudioInformation)) { - TArray TempData; - TempData.Append(&Buffer[0], GetSampleCount(AudioInformation)); - OnChunkProcessed.ExecuteIfBound(TempData, AudioInformation); - TempData.Empty(); - TempData.Append(&Buffer[GetSampleCount(AudioInformation)], Buffer.Num() - GetSampleCount(AudioInformation)); - Buffer.Empty(); + TArray TempData; + while ((uint32)Buffer.Num() > SampleCount) { + TempData.Reset(); + TempData.Append(&Buffer[0], SampleCount); + OnChunkProcessed.ExecuteIfBound(TempData, AudioInformation, ESTTChainState::Processing); + bHasFilledFully = true; + TempData.Reset(); + TempData.Append(&Buffer[SampleCount], Buffer.Num() - SampleCount); + Buffer.Reset(); Buffer.Append(TempData); } - } + } } uint32 USTTPreprocessorBuffer::GetSampleCount(FAudioInformation AudioInformation) { return (uint32)((AudioInformation.SampleRate) / 1000 * (float)BufferSize); } - -void USTTPreprocessorBuffer::OnSpeechChanged(ESTTTalkingState NewSpeechState) -{ - if (NewSpeechState == ESTTTalkingState::SILENCE || NewSpeechState == ESTTTalkingState::TRANSCRIBING) { - OnChunkProcessed.ExecuteIfBound(Buffer, BufferAudioInformation); - Buffer.Empty(); - } -} diff --git a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Preprocessor/STTPreprocessorConverter.cpp b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Preprocessor/STTPreprocessorConverter.cpp index bf4fe8a..e259be5 100644 --- a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Preprocessor/STTPreprocessorConverter.cpp +++ b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Preprocessor/STTPreprocessorConverter.cpp @@ -4,12 +4,14 @@ #include "Preprocessor/STTPreprocessorConverter.h" #include "STTManagerBase.h" -void USTTPreprocessorConverter::OnChunkReceived(TArray PCMData, FAudioInformation AudioInformation) -{ - // Check if input data is valid +void USTTPreprocessorConverter::OnChunkReceived(TArray PCMData, FAudioInformation AudioInformation, ESTTChainState ChainState) +{ if (PCMData.Num() == 0) { - STTManager->OnSTTError.Broadcast(TEXT("STTPreprocessorConverter: Received empty PCM data.")); + if (ChainState != ESTTChainState::Processing) + OnChunkProcessed.ExecuteIfBound({}, AudioInformation, ChainState); + else + STTManager->OnSTTError.Broadcast(TEXT("STTPreprocessorConverter: Received empty PCM data.")); return; } @@ -83,11 +85,11 @@ void USTTPreprocessorConverter::OnChunkReceived(TArray PCMData, FAudioInf return; } // Pass the resampled data to the next processor in the chain - OnChunkProcessed.ExecuteIfBound(ResampledPCMData, TargetAudioInformation); + OnChunkProcessed.ExecuteIfBound(ResampledPCMData, TargetAudioInformation, ChainState); } else { // No resampling needed, pass the converted data as is // Log some final sample values - OnChunkProcessed.ExecuteIfBound(ConvertedPCMData, TargetAudioInformation); + OnChunkProcessed.ExecuteIfBound(ConvertedPCMData, TargetAudioInformation, ChainState); } } diff --git a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Preprocessor/STTPreprocessorDebugger.cpp b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Preprocessor/STTPreprocessorDebugger.cpp index 7634ef3..22fa4f2 100644 --- a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Preprocessor/STTPreprocessorDebugger.cpp +++ b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Preprocessor/STTPreprocessorDebugger.cpp @@ -21,7 +21,7 @@ void USTTPreprocessorDebugger::InitSTTPreprocessor(USTTManagerBase* BaseSTTManag DataBytesWritten = 0; } -void USTTPreprocessorDebugger::OnChunkReceived(TArray PCMData, FAudioInformation AudioInformation) +void USTTPreprocessorDebugger::OnChunkReceived(TArray PCMData, FAudioInformation AudioInformation, ESTTChainState ChainState) { STTManager->OnSTTLog.Broadcast(FString::Printf(TEXT("Audio Chunks passing through the STTPreprocessorDebugger. %i PCM int16 Array entries."), PCMData.Num())); @@ -50,7 +50,7 @@ void USTTPreprocessorDebugger::OnChunkReceived(TArray PCMData, FAudioInfo DataBytesWritten += static_cast(NumBytes); } - OnChunkProcessed.ExecuteIfBound(PCMData, AudioInformation); + OnChunkProcessed.ExecuteIfBound(PCMData, AudioInformation, ChainState); } void USTTPreprocessorDebugger::DestroySTTPreprocessor() diff --git a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Preprocessor/STTPreprocessorPTT.cpp b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Preprocessor/STTPreprocessorPTT.cpp index 8455833..0a90369 100644 --- a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Preprocessor/STTPreprocessorPTT.cpp +++ b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Preprocessor/STTPreprocessorPTT.cpp @@ -6,6 +6,7 @@ void USTTPreprocessorPTT::InitSTTPreprocessor(USTTManagerBase* BaseSTTManager, FSTTBaseSettings InSTTBaseSettings, bool InDebugMode) { USTTPreprocessorBase::InitSTTPreprocessor(BaseSTTManager, InSTTBaseSettings, InDebugMode); + BaseSTTManager->OnSTTButtonStateChanged.AddUniqueDynamic(this, &USTTPreprocessorPTT::OnPTTStateChanged); BaseSTTManager->OnSpeechStateChanged.AddUniqueDynamic(this, &USTTPreprocessorPTT::OnUserSpeechStateChanged); } @@ -14,14 +15,29 @@ void USTTPreprocessorPTT::DestroySTTPreprocessor() } +void USTTPreprocessorPTT::OnChunkReceived(TArray PCMData, FAudioInformation AudioInformation, ESTTChainState ChainState) +{ + if (!PTTPressed) + return; + + LastAudioInformation = AudioInformation; + OnChunkProcessed.ExecuteIfBound(PCMData, AudioInformation, ESTTChainState::Processing); +} -void USTTPreprocessorPTT::OnChunkReceived(TArray PCMData, FAudioInformation AudioInformation) +void USTTPreprocessorPTT::OnPTTStateChanged(bool IsPressed) { - if(NewSpeechState == ESTTTalkingState::TALKING) - OnChunkProcessed.ExecuteIfBound(PCMData, AudioInformation); + PTTPressed = IsPressed; + + if (!IsPressed) + OnChunkProcessed.ExecuteIfBound({}, LastAudioInformation, ESTTChainState::Finalizing); + } -void USTTPreprocessorPTT::OnUserSpeechStateChanged(ESTTTalkingState InNewSpeechState) +void USTTPreprocessorPTT::OnUserSpeechStateChanged(ESTTTalkingState NewSpeechState) { - NewSpeechState = InNewSpeechState; + if(NewSpeechState==ESTTTalkingState::BLOCKED) + { + PTTPressed = false; + OnChunkProcessed.ExecuteIfBound({}, LastAudioInformation, ESTTChainState::Discarding); + } } diff --git a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Preprocessor/STTPreprocessorSpeexDSP.cpp b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Preprocessor/STTPreprocessorSpeexDSP.cpp index 528dd43..ed13b7c 100644 --- a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Preprocessor/STTPreprocessorSpeexDSP.cpp +++ b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Preprocessor/STTPreprocessorSpeexDSP.cpp @@ -39,17 +39,19 @@ void USTTPreprocessorSpeexDSP::PostInitProperties() Super::PostInitProperties(); } -void USTTPreprocessorSpeexDSP::OnChunkReceived(TArray PCMData, FAudioInformation AudioInformation) +void USTTPreprocessorSpeexDSP::OnChunkReceived(TArray PCMData, FAudioInformation AudioInformation, ESTTChainState ChainState) { - // Expect 16 kHz mono; chunk size may be 30 ms (480 samples). We process in 10 ms frames (160 samples) - if (PCMData.Num() % FrameSizeSamples != 0) + if (PCMData.Num() == 0 || PCMData.Num() % FrameSizeSamples != 0) { - UE_LOG(LogTemp, Warning, TEXT("PCMData size (%d) is not a multiple of frame size (%d)!"), PCMData.Num(), FrameSizeSamples); + if (ChainState != ESTTChainState::Processing) + OnChunkProcessed.ExecuteIfBound(PCMData, AudioInformation, ChainState); + else + UE_LOG(LogTemp, Warning, TEXT("PCMData size (%d) is not a multiple of frame size (%d)!"), PCMData.Num(), FrameSizeSamples); return; } int vadReturnvalue = 0; if (vadReturnvalue > 1 || m_vad < 1) { - OnChunkProcessed.ExecuteIfBound(PCMData, AudioInformation); + OnChunkProcessed.ExecuteIfBound(PCMData, AudioInformation, ChainState); } } diff --git a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Preprocessor/STTPreprocessorVAD.cpp b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Preprocessor/STTPreprocessorVAD.cpp index f942ab1..db85d06 100644 --- a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Preprocessor/STTPreprocessorVAD.cpp +++ b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Preprocessor/STTPreprocessorVAD.cpp @@ -17,10 +17,15 @@ void USTTPreprocessorVAD::DestroySTTPreprocessor() fvad = nullptr; } -void USTTPreprocessorVAD::OnChunkReceived(TArray PCMData, FAudioInformation AudioInformation) +void USTTPreprocessorVAD::OnChunkReceived(TArray PCMData, FAudioInformation AudioInformation, ESTTChainState ChainState) { if(STTBaseSettings.bUsePTT) - OnChunkProcessed.ExecuteIfBound(PCMData, AudioInformation); + { + OnChunkProcessed.ExecuteIfBound(PCMData, AudioInformation, ChainState); + return; + } + + LastAudioInformation = AudioInformation; if (!fvad) { @@ -61,8 +66,6 @@ void USTTPreprocessorVAD::OnChunkReceived(TArray PCMData, FAudioInformati const float MeanSquares = static_cast(SumSquares / static_cast(NumFrames)); const float Rms = FMath::Sqrt(FMath::Max(MeanSquares, 1.e-12f)); const float Dbfs = 20.0f * FMath::LogX(10.0f, Rms); - //UE_LOG(LogTemp, Warning, TEXT("Dbfs %f"), Dbfs); - isLoadEnough = (Dbfs < static_cast(STTBaseSettings.VADSettings.VAD_MinSpeechAmplitude)); isLoadEnough = (Dbfs > static_cast(STTBaseSettings.VADSettings.VAD_MinSpeechAmplitude)); } @@ -83,31 +86,32 @@ void USTTPreprocessorVAD::OnChunkReceived(TArray PCMData, FAudioInformati case -1: STTManager->OnSTTError.Broadcast(FString::Printf(TEXT("Invalid frame length %i entries in buffer"), PCMData.Num())); break; - case 0: + case 0: if (talkingState == ESTTTalkingState::TALKING) { - if (timeInStateInSeconds > STTBaseSettings.FreespeechPostRollTime) { - STTManager->UserSpeechStateChanged(ESTTTalkingState::SILENCE); + if (timeInStateInSeconds > STTBaseSettings.FreespeechPostRollTime) { + OnChunkProcessed.ExecuteIfBound(PCMData, AudioInformation, ESTTChainState::Finalizing); + STTManager->UserSpeechStateChanged(ESTTTalkingState::SILENCE); // UI only } else { - OnChunkProcessed.ExecuteIfBound(PCMData, AudioInformation); //Even send silence if threshold is not met - } + OnChunkProcessed.ExecuteIfBound(PCMData, AudioInformation, ESTTChainState::Processing); //Even send silence if threshold is not met + } } break; case 1: - if (talkingState == ESTTTalkingState::BLOCKED) { + if (talkingState == ESTTTalkingState::BLOCKED) { if(timeInStateInSeconds > STTBaseSettings.VADSettings.VAD_SpeechWhileBlocked) { STTManager->OnSpeechDetectedWhileBlocked.Broadcast(); - if (STTBaseSettings.bCanInterrupt) + if (STTBaseSettings.bCanInterrupt) { STTManager->SetBlocked(false); if (Buffer.Num() > 0) { Buffer.Append(PCMData); - OnChunkProcessed.ExecuteIfBound(Buffer, AudioInformation); + OnChunkProcessed.ExecuteIfBound(Buffer, AudioInformation, ESTTChainState::Processing); Buffer.Empty(); } - } + } } else Buffer.Append(PCMData); //Buffer Data @@ -120,15 +124,15 @@ void USTTPreprocessorVAD::OnChunkReceived(TArray PCMData, FAudioInformati if(Buffer.Num() > 0) { Buffer.Append(PCMData); - OnChunkProcessed.ExecuteIfBound(Buffer, AudioInformation); + OnChunkProcessed.ExecuteIfBound(Buffer, AudioInformation, ESTTChainState::Processing); Buffer.Empty(); - } + } } else Buffer.Append(PCMData); //Buffer Data } else - OnChunkProcessed.ExecuteIfBound(PCMData, AudioInformation); + OnChunkProcessed.ExecuteIfBound(PCMData, AudioInformation, ESTTChainState::Processing); break; } } @@ -140,5 +144,6 @@ void USTTPreprocessorVAD::OnUserSpeechStateChanged(ESTTTalkingState NewSpeechSta if (NewSpeechState == ESTTTalkingState::BLOCKED) { Buffer.Empty(); - } + OnChunkProcessed.ExecuteIfBound({}, LastAudioInformation, ESTTChainState::Discarding); + } } diff --git a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Preprocessor/STTPreprocessorWebRTC.cpp b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Preprocessor/STTPreprocessorWebRTC.cpp index f599fa0..8b1ebfb 100644 --- a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Preprocessor/STTPreprocessorWebRTC.cpp +++ b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Preprocessor/STTPreprocessorWebRTC.cpp @@ -144,10 +144,13 @@ void USTTPreprocessorWebRTC::PostInitProperties() } void USTTPreprocessorWebRTC::OnChunkReceived(TArray PCMData, - FAudioInformation AudioInformation) + FAudioInformation AudioInformation, + ESTTChainState ChainState) { - if (PCMData.Num() <= 0) + if (PCMData.Num() == 0) { + if (ChainState != ESTTChainState::Processing) + OnChunkProcessed.ExecuteIfBound({}, AudioInformation, ChainState); return; } // Remember the *actual* capture format once @@ -184,7 +187,7 @@ void USTTPreprocessorWebRTC::OnChunkReceived(TArray PCMData, { // Forward as 48 kHz; downstream converter can resample to 16k or whatever is needed AudioInformation.SampleRate = WebRTCChannel.GetSampleRate(); - OnChunkProcessed.ExecuteIfBound(Processed, AudioInformation); + OnChunkProcessed.ExecuteIfBound(Processed, AudioInformation, ChainState); } } diff --git a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Processor/Azure/AzureRunnable.cpp b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Processor/Azure/AzureRunnable.cpp index a47b3ff..13b87d0 100644 --- a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Processor/Azure/AzureRunnable.cpp +++ b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Processor/Azure/AzureRunnable.cpp @@ -30,6 +30,9 @@ bool FAzureRunnable::Init() SpeechInputStream = SpeechSDK::Audio::AudioInputStream::CreatePushStream(); AudioConfig = SpeechSDK::Audio::AudioConfig::FromStreamInput(SpeechInputStream); + // Add before Recognizer = SpeechSDK::SpeechRecognizer::FromConfig(...) + Config->SetProperty(SpeechSDK::PropertyId::Speech_SegmentationSilenceTimeoutMs, "700"); + try { Recognizer = SpeechSDK::SpeechRecognizer::FromConfig(Config, AudioConfig); @@ -102,6 +105,14 @@ bool FAzureRunnable::Init() }); }); + Recognizer->Canceled.Connect([WeakOwner, Self](const auto& EventArgs) { + FString Reason = UTF8_TO_TCHAR(EventArgs.ErrorDetails.c_str()); + AsyncTask(ENamedThreads::GameThread, [WeakOwner, Reason, Self]() { + if (WeakOwner.IsValid()) + WeakOwner->OnAzureError(FString::Printf(TEXT("Canceled: %s"), *Reason)); + }); + }); + return Recognizer != nullptr; } diff --git a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Processor/Azure/STTProcessorAzure.cpp b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Processor/Azure/STTProcessorAzure.cpp index b88d9ca..0453a83 100644 --- a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Processor/Azure/STTProcessorAzure.cpp +++ b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Processor/Azure/STTProcessorAzure.cpp @@ -39,12 +39,12 @@ void USTTProcessorAzure::InitSTTProcessor(USTTManagerBase* BaseSTTManager, USTTB // Set properties FString tmpAzureLanguages = ""; - for (int i = 0; i < AzureProcessorConfig->AzureLanguages.Num(); i++) + for (int i = 0; i < AzureProcessorConfig->BaseSettings.STTLanguages.Num(); i++) { if (i > 0) { tmpAzureLanguages += ","; - } - tmpAzureLanguages += USTTProcessorAzure::AzureEnumToString(AzureProcessorConfig->AzureLanguages[i]); + } + tmpAzureLanguages += USTTProcessorAzure::AzureEnumToString(AzureProcessorConfig->BaseSettings.STTLanguages[i]); } std::string LanguageString = TCHAR_TO_UTF8(*tmpAzureLanguages); config->SetProperty(SpeechSDK::PropertyId::SpeechServiceConnection_AutoDetectSourceLanguages, LanguageString); @@ -68,47 +68,34 @@ void USTTProcessorAzure::DestroySTTProcessor() // Stops recognition. } -void USTTProcessorAzure::OnChunkReceived(TArray PCMData, FAudioInformation AudioInformation) +void USTTProcessorAzure::OnChunkReceived(TArray PCMData, FAudioInformation AudioInformation, ESTTChainState ChainState) { - if (IsValid(STTManager) && STTManager->IsBlocked()) - return; + LastChainState = ChainState; - if (!AzureRunnable || !bTranscriptionRunning) //Runable not ready or previous session ended + if (ChainState == ESTTChainState::Discarding) { - USTTProcessorAzure::StartRecognition(); + StopRecognition(true); + return; } - // Get the pointer to the raw PCM data - int16* rawPCMData = PCMData.GetData(); - // Calculate the size of the data buffer in bytes - uint32_t bufferSize = PCMData.Num() * sizeof(int16); - // Write the raw PCM data to the PushAudioInputStream, preserving const qualifier - AzureRunnable->AddAudioChunk(PCMData); + if (PCMData.Num() > 0) + { + if (!AzureRunnable || !bTranscriptionRunning) + USTTProcessorAzure::StartRecognition(); + AzureRunnable->AddAudioChunk(PCMData); + } + + if (ChainState == ESTTChainState::Finalizing) + StopRecognition(false); } -void USTTProcessorAzure::ChangeAzureLanguage(TArray InAzureLanguages) +void USTTProcessorAzure::ChangeAzureLanguage(TArray InLanguages) { - AzureProcessorConfig->AzureLanguages = InAzureLanguages; + AzureProcessorConfig->BaseSettings.STTLanguages = InLanguages; if (bDebugMode && IsValid(STTManager)) - STTManager->OnSTTLog.Broadcast(TEXT("Azure languages changed.")); + STTManager->OnSTTLog.Broadcast(TEXT("Azure languages changed.")); } -void USTTProcessorAzure::OnSpeechStateChanged(ESTTTalkingState TalkingState) -{ - if (TalkingState == ESTTTalkingState::BLOCKED) { - StopRecognition(true); - } - else if (TalkingState == ESTTTalkingState::SILENCE || TalkingState == ESTTTalkingState::TRANSCRIBING) { - if (AzureRunnable) { - StopRecognition(false); // Signal stop, runnable delivers final result via OnRecognized/OnRunnableEnded - } - else if (!intermediateResult.IsEmpty()) { - // No runnable pending, send accumulated result immediately - USTTProcessorBase::OnTranscriptionResult(TranscriptionCounter, intermediateResult, DetectedLanguage); - intermediateResult = ""; - } - } -} void USTTProcessorAzure::StartRecognition() { @@ -158,7 +145,7 @@ void USTTProcessorAzure::OnRecognized(const FString& RecognizedText, const FStri } if (!IsValid(STTManager)) return; - if (STTManager->IsBlocked()) + if (STTManager->IsBlocked() || LastChainState == ESTTChainState::Discarding) return; this->DetectedLanguage = Language; @@ -167,7 +154,7 @@ void USTTProcessorAzure::OnRecognized(const FString& RecognizedText, const FStri intermediateResult += " " + RecognizedText; else intermediateResult = RecognizedText; - if (STTManager->GetCurrentSpeechState() == ESTTTalkingState::TALKING) { //User still talking + if (LastChainState == ESTTChainState::Processing) { //User still talking USTTProcessorBase::OnTranscriptionIntermediateResult(TranscriptionCounter, *intermediateResult); } else { @@ -185,8 +172,8 @@ void USTTProcessorAzure::OnConnectionSuccess() // Connection test runnable returns from Run() before posting this callback, // so Run() is already done — direct null is safe. AzureRunnable = nullptr; + STTManager->OnSTTLog.Broadcast(TEXT("STTProcessor Azure Speech initialized successfully.")); STTManager->OnReady.Broadcast(); - STTManager->OnSpeechStateChanged.AddUniqueDynamic(this, &USTTProcessorAzure::OnSpeechStateChanged); } void USTTProcessorAzure::OnRunnableEnded(FAzureRunnable* Caller) @@ -204,7 +191,7 @@ void USTTProcessorAzure::OnRunnableEnded(FAzureRunnable* Caller) intermediateResult.Empty(); } - if (!STTManager->IsBlocked()) + if (LastChainState == ESTTChainState::Finalizing && IsValid(STTManager) && !STTManager->IsBlocked()) STTManager->UserSpeechStateChanged(ESTTTalkingState::SILENCE); } } @@ -220,7 +207,7 @@ void USTTProcessorAzure::OnRunnableEnded(FAzureRunnable* Caller) intermediateResult.Empty(); } - if (!STTManager->IsBlocked()) + if (LastChainState == ESTTChainState::Finalizing && IsValid(STTManager) && !STTManager->IsBlocked()) STTManager->UserSpeechStateChanged(ESTTTalkingState::SILENCE); } } @@ -240,54 +227,58 @@ void USTTProcessorAzure::OnAzureError(FString Error) if (IsValid(STTManager)) { STTManager->OnSTTError.Broadcast(Error); - STTManager->UserSpeechStateChanged(ESTTTalkingState::SILENCE); + if(!STTManager->IsBlocked()) + STTManager->UserSpeechStateChanged(ESTTTalkingState::SILENCE); } } -FString USTTProcessorAzure::AzureEnumToString(EAzureLanguages Language) +FString USTTProcessorAzure::AzureEnumToString(ESTTLanguage Language) { switch (Language) { - case EAzureLanguages::German_Germany: - return "de-DE"; - break; - case EAzureLanguages::English_UK: - return "en-GB"; - break; - case EAzureLanguages::English_India: - return "en-IN"; - break; - case EAzureLanguages::English_US: - return "en-US"; - break; - case EAzureLanguages::Spanish_Spain: - return "es-ES"; - break; - case EAzureLanguages::Spanish_Mexico: - return "es-MX"; - break; - case EAzureLanguages::French_France: - return "fr-FR"; - break; - case EAzureLanguages::Hindi_India: - return "hi-IN"; - break; - case EAzureLanguages::Italian_Italy: - return "it-IT"; - break; - case EAzureLanguages::Japanese_Japan: - return "ja-JP"; - break; - case EAzureLanguages::Korean_Korea: - return "ko-KR"; - break; - case EAzureLanguages::Portuguese_Brazil: - return "pt-BR"; - break; - case EAzureLanguages::Chinese_Simplified: - return "zh-CN"; - break; + case ESTTLanguage::en: return "en-US"; + case ESTTLanguage::de: return "de-DE"; + case ESTTLanguage::fr: return "fr-FR"; + case ESTTLanguage::es: return "es-ES"; + case ESTTLanguage::pt: return "pt-BR"; + case ESTTLanguage::zh: return "zh-CN"; + case ESTTLanguage::ja: return "ja-JP"; + case ESTTLanguage::hi: return "hi-IN"; + case ESTTLanguage::it: return "it-IT"; + case ESTTLanguage::ko: return "ko-KR"; + case ESTTLanguage::nl: return "nl-NL"; + case ESTTLanguage::pl: return "pl-PL"; + case ESTTLanguage::ru: return "ru-RU"; + case ESTTLanguage::sv: return "sv-SE"; + case ESTTLanguage::tr: return "tr-TR"; + case ESTTLanguage::tl: return "fil-PH"; + case ESTTLanguage::bg: return "bg-BG"; + case ESTTLanguage::ro: return "ro-RO"; + case ESTTLanguage::ar: return "ar-SA"; + case ESTTLanguage::cs: return "cs-CZ"; + case ESTTLanguage::el: return "el-GR"; + case ESTTLanguage::fi: return "fi-FI"; + case ESTTLanguage::hr: return "hr-HR"; + case ESTTLanguage::ms: return "ms-MY"; + case ESTTLanguage::sk: return "sk-SK"; + case ESTTLanguage::da: return "da-DK"; + case ESTTLanguage::ta: return "ta-IN"; + case ESTTLanguage::uk: return "uk-UA"; + case ESTTLanguage::hu: return "hu-HU"; + case ESTTLanguage::no: return "nb-NO"; + case ESTTLanguage::vi: return "vi-VN"; + case ESTTLanguage::bn: return "bn-IN"; + case ESTTLanguage::th: return "th-TH"; + case ESTTLanguage::he: return "he-IL"; + case ESTTLanguage::ka: return "ka-GE"; + case ESTTLanguage::id: return "id-ID"; + case ESTTLanguage::te: return "te-IN"; + case ESTTLanguage::gu: return "gu-IN"; + case ESTTLanguage::kn: return "kn-IN"; + case ESTTLanguage::ml: return "ml-IN"; + case ESTTLanguage::mr: return "mr-IN"; + case ESTTLanguage::pa: return "pa-IN"; + default: return "UNDEFINED"; } - return "UNDEFINED"; } diff --git a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Processor/Parakeet/STTParakeetProcessorBase.cpp b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Processor/Parakeet/STTParakeetProcessorBase.cpp index 612ed92..2efdcac 100644 --- a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Processor/Parakeet/STTParakeetProcessorBase.cpp +++ b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Processor/Parakeet/STTParakeetProcessorBase.cpp @@ -36,9 +36,15 @@ void USTTParakeetProcessorBase::InitSTTProcessor(USTTManagerBase* BaseSTTManager } #if WITH_EDITOR - bIsEditor = GIsEditor; + if (ParakeetConfig->KeepAliveRule == ESTTKeepAliveRule::Never) + bKeepAlive = false; + else + bKeepAlive = true; #else - bIsEditor = false; + if (ParakeetConfig->KeepAliveRule == ESTTKeepAliveRule::Always) + bKeepAlive = true; + else + bKeepAlive = false; #endif // Resolve batch file path @@ -82,7 +88,7 @@ void USTTParakeetProcessorBase::InitSTTProcessor(USTTManagerBase* BaseSTTManager // Editor: try to connect to existing instance (with brief delay for server to be ready after previous disconnect) // Non-editor: kill stale processes and launch fresh - if (!bIsEditor) + if (!bKeepAlive) { KillExistingParakeetProcesses(); if (UWorld* World = STTManager->GetWorld()) @@ -127,7 +133,6 @@ void USTTParakeetProcessorBase::DestroySTTProcessor() // Unbind delegate if (IsValid(STTManager)) { - STTManager->OnSpeechStateChanged.RemoveDynamic(this, &USTTParakeetProcessorBase::OnSpeechStateChanged); if (UWorld* World = STTManager->GetWorld()) { @@ -144,7 +149,7 @@ void USTTParakeetProcessorBase::DestroySTTProcessor() } // Kill local process if we spawned it (non-editor only) - if (!bIsEditor && bLaunchedLocalServer) + if (!bKeepAlive && bLaunchedLocalServer) { if (ParakeetProcHandle.IsValid()) { @@ -154,7 +159,7 @@ void USTTParakeetProcessorBase::DestroySTTProcessor() bLaunchedLocalServer = false; } - if (!bIsEditor) + if (!bKeepAlive) { KillExistingParakeetProcesses(); } @@ -171,12 +176,15 @@ void USTTParakeetProcessorBase::DestroySTTProcessor() // Audio chunk handling // --------------------------------------------------------------------------- -void USTTParakeetProcessorBase::OnChunkReceived(TArray PCMData, FAudioInformation AudioInformation) +void USTTParakeetProcessorBase::OnChunkReceived(TArray PCMData, FAudioInformation AudioInformation, ESTTChainState ChainState) { - if (IsValid(STTManager) && STTManager->IsBlocked()) + if (ChainState == ESTTChainState::Discarding) { - if (bDebugMode && IsValid(STTManager)) - STTManager->OnSTTLog.Broadcast(TEXT("Parakeet: OnChunkReceived skipped (blocked)")); + TSharedPtr Obj = MakeShared(); + Obj->SetStringField(TEXT("type"), TEXT("clear")); + SendJsonMessage(Obj); + IntermediateResult.Empty(); + bTranscriptionRunning = false; return; } @@ -187,22 +195,27 @@ void USTTParakeetProcessorBase::OnChunkReceived(TArray PCMData, FAudioInf return; } - // Encode PCM16 data as base64 - TArray RawBytes; - RawBytes.Append(reinterpret_cast(PCMData.GetData()), PCMData.Num() * sizeof(int16)); - FString B64 = FBase64::Encode(RawBytes); + if (PCMData.Num() > 0) + { + // Encode PCM16 data as base64 + TArray RawBytes; + RawBytes.Append(reinterpret_cast(PCMData.GetData()), PCMData.Num() * sizeof(int16)); + FString B64 = FBase64::Encode(RawBytes); - // Build JSON message - TSharedPtr Obj = MakeShared(); - Obj->SetStringField(TEXT("type"), TEXT("audio")); - Obj->SetStringField(TEXT("data_b64"), B64); + TSharedPtr Obj = MakeShared(); + Obj->SetStringField(TEXT("type"), TEXT("audio")); + Obj->SetStringField(TEXT("data_b64"), B64); + SendJsonMessage(Obj); - SendJsonMessage(Obj); + if (!bTranscriptionRunning) + USTTProcessorBase::OnTranscriptionStarted(); + } - // Start transcription tracking if not already running - if (!bTranscriptionRunning) + if (ChainState == ESTTChainState::Finalizing) { - USTTProcessorBase::OnTranscriptionStarted(); + TSharedPtr Obj = MakeShared(); + Obj->SetStringField(TEXT("type"), TEXT("finalize")); + SendJsonMessage(Obj); } } @@ -210,43 +223,6 @@ void USTTParakeetProcessorBase::OnChunkReceived(TArray PCMData, FAudioInf // Speech state handling // --------------------------------------------------------------------------- -void USTTParakeetProcessorBase::OnSpeechStateChanged(ESTTTalkingState TalkingState) -{ - if (!STTManager->IsSTTFullyInitialized()) - return; - - if (bDebugMode && IsValid(STTManager)) - STTManager->OnSTTLog.Broadcast(FString::Printf(TEXT("Parakeet: OnSpeechStateChanged -> %d"), (int32)TalkingState)); - - if (TalkingState == ESTTTalkingState::BLOCKED) - { - // Discard everything - TSharedPtr Obj = MakeShared(); - Obj->SetStringField(TEXT("type"), TEXT("clear")); - SendJsonMessage(Obj); - IntermediateResult.Empty(); - bTranscriptionRunning = false; - } - else if (TalkingState == ESTTTalkingState::SILENCE || TalkingState == ESTTTalkingState::TRANSCRIBING) - { - if (bTranscriptionRunning) - { - // Request final transcription - TSharedPtr Obj = MakeShared(); - Obj->SetStringField(TEXT("type"), TEXT("finalize")); - SendJsonMessage(Obj); - } - else - { - // If we have accumulated intermediate text but transcription already ended - if (!IntermediateResult.IsEmpty()) - { - USTTProcessorBase::OnTranscriptionIntermediateResult(TranscriptionCounter, IntermediateResult); - IntermediateResult.Empty(); - } - } - } -} // --------------------------------------------------------------------------- // Callbacks from FParakeetRunnable (game thread) @@ -254,13 +230,12 @@ void USTTParakeetProcessorBase::OnSpeechStateChanged(ESTTTalkingState TalkingSta void USTTParakeetProcessorBase::OnParakeetReady() { - if (bDebugMode && IsValid(STTManager)) - STTManager->OnSTTLog.Broadcast(TEXT("Parakeet server READY")); + if (IsValid(STTManager)) + STTManager->OnSTTLog.Broadcast(TEXT("STTManager Parakeet initialized successfully. Server READY")); if (IsValid(STTManager)) { STTManager->OnSTTFullyInitialized(); - STTManager->OnSpeechStateChanged.AddUniqueDynamic(this, &USTTParakeetProcessorBase::OnSpeechStateChanged); } } @@ -466,7 +441,7 @@ bool USTTParakeetProcessorBase::StartParakeetProcess() // Python side when the UObject (and its pipe-reader thread) is destroyed // at the end of a PIE session while the Python process is kept alive for // reconnect, which stalls the transcription thread on subsequent connects. - if (bIsEditor) + if (bKeepAlive) { ParakeetProcHandle = FPlatformProcess::CreateProc(*ParakeetBatPath, *Args, false, !bDebugMode, !bDebugMode, &ProcId, 0, nullptr, nullptr, nullptr); if (!ParakeetProcHandle.IsValid()) @@ -493,7 +468,7 @@ bool USTTParakeetProcessorBase::StartParakeetProcess() FPlatformProcess::CreatePipe(StdErrReadPipe, StdErrWritePipe); //Removed the piping for DebugMode - especially in edtior build pipes break after first connect - ParakeetProcHandle = FPlatformProcess::CreateProc(*ParakeetBatPath, *Args, false, !(bDebugMode && bIsEditor), !(bDebugMode && bIsEditor), &ProcId, 0, nullptr, (bDebugMode && bIsEditor) ? nullptr : StdOutWritePipe, bDebugMode ? nullptr : StdErrWritePipe); + ParakeetProcHandle = FPlatformProcess::CreateProc(*ParakeetBatPath, *Args, false, !(bDebugMode && bKeepAlive), !(bDebugMode && bKeepAlive), &ProcId, 0, nullptr, (bDebugMode && bKeepAlive) ? nullptr : StdOutWritePipe, bDebugMode ? nullptr : StdErrWritePipe); if (!ParakeetProcHandle.IsValid()) { if (IsValid(STTManager)) diff --git a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Processor/RealtimeAPI/STTProcessorRealtimeAPI.cpp b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Processor/RealtimeAPI/STTProcessorRealtimeAPI.cpp index 1c6799d..1c926e5 100644 --- a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Processor/RealtimeAPI/STTProcessorRealtimeAPI.cpp +++ b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Processor/RealtimeAPI/STTProcessorRealtimeAPI.cpp @@ -24,8 +24,11 @@ void USTTProcessorRealtimeAPI::ClearToRealtimeAPI() AvatarCoreAIRealtime = nullptr; } -void USTTProcessorRealtimeAPI::OnChunkReceived(TArray PCMData, FAudioInformation AudioInformation) +void USTTProcessorRealtimeAPI::OnChunkReceived(TArray PCMData, FAudioInformation AudioInformation, ESTTChainState ChainState) { + if (ChainState == ESTTChainState::Discarding) + return; + if (AvatarCoreAIRealtime != nullptr) AvatarCoreAIRealtime->OnSTTAudioChunk(PCMData); } \ No newline at end of file diff --git a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Processor/STTProcessorDebugSaveWav.cpp b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Processor/STTProcessorDebugSaveWav.cpp index 7b71382..8ddafda 100644 --- a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Processor/STTProcessorDebugSaveWav.cpp +++ b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Processor/STTProcessorDebugSaveWav.cpp @@ -13,8 +13,13 @@ void USTTProcessorDebugSaveWav::DestroySTTProcessor() SaveWave(FilePath); } -void USTTProcessorDebugSaveWav::OnChunkReceived(TArray PCMData, FAudioInformation AudioInformation) +void USTTProcessorDebugSaveWav::OnChunkReceived(TArray PCMData, FAudioInformation AudioInformation, ESTTChainState ChainState) { + if (ChainState == ESTTChainState::Discarding) + { + StoredPCMData.Empty(); + return; + } StoredAudioInformation = AudioInformation; StoredPCMData.Append(PCMData); } diff --git a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Processor/Whisper/STTProcessorWhisper.cpp b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Processor/Whisper/STTProcessorWhisper.cpp index 5622f20..9cd7eeb 100644 --- a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Processor/Whisper/STTProcessorWhisper.cpp +++ b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Processor/Whisper/STTProcessorWhisper.cpp @@ -59,19 +59,15 @@ void USTTProcessorWhisper::InitSTTProcessor(USTTManagerBase* BaseSTTManager, UST NormalizeWhisperURL(); - if (IsValid(STTManager)) - { - STTManager->OnSpeechStateChanged.AddUniqueDynamic(this, &USTTProcessorWhisper::OnSpeechStateChanged); - } - PerformHealthCheck(); + + STTManager->OnSTTLog.Broadcast(FString::Printf(TEXT("STTProcessor OpenAI %s initialized successfully."), *TranscribeModelEnumToString(WhisperProcessorConfig->Model))); } void USTTProcessorWhisper::ClearSTTProcessor() { USTTProcessorBase::ClearSTTProcessor(); BufferedPCMData.Empty(); - bHasBufferedAudioInformation = false; for (TSharedPtr& Request : ActiveRequests) { @@ -90,92 +86,28 @@ void USTTProcessorWhisper::DestroySTTProcessor() STTManager = nullptr; } -void USTTProcessorWhisper::OnChunkReceived(TArray PCMData, FAudioInformation AudioInformation) +void USTTProcessorWhisper::OnChunkReceived(TArray PCMData, FAudioInformation AudioInformation, ESTTChainState ChainState) { - if (CurrentTalkingState != ESTTTalkingState::TALKING) - return; - - if (PCMData.Num() == 0) - return; - - if (!bHasBufferedAudioInformation) - { - BufferedAudioInformation = AudioInformation; - bHasBufferedAudioInformation = true; - } - else if (BufferedAudioInformation.SampleRate != AudioInformation.SampleRate || - BufferedAudioInformation.NumChannels != AudioInformation.NumChannels) - { - BufferedPCMData.Empty(); - BufferedAudioInformation = AudioInformation; - } - - const int64 BytesPerSample = sizeof(int16); - const int64 CurrentBytes = static_cast(BufferedPCMData.Num()) * BytesPerSample; - const int64 NewBytes = static_cast(PCMData.Num()) * BytesPerSample; - const int64 MaxUploadBytes = static_cast(25) * 1024 * 1024; - const int64 MaxAudioBytes = MaxUploadBytes - 1024; - - if (CurrentBytes + NewBytes > MaxAudioBytes) - { - int64 ExcessBytes = CurrentBytes + NewBytes - MaxAudioBytes; - int64 SamplesToRemove = (ExcessBytes + BytesPerSample - 1) / BytesPerSample; - - if (SamplesToRemove >= BufferedPCMData.Num()) - { - BufferedPCMData.Empty(); - if (NewBytes > MaxAudioBytes) - { - int64 NewSamplesAllowed = MaxAudioBytes / BytesPerSample; - if (NewSamplesAllowed > 0 && NewSamplesAllowed < PCMData.Num()) - { - int32 StartIndex = PCMData.Num() - static_cast(NewSamplesAllowed); - BufferedPCMData.Append(&PCMData[StartIndex], static_cast(NewSamplesAllowed)); - } - } - else - { - BufferedPCMData.Append(PCMData); - } - } - else - { - BufferedPCMData.RemoveAt(0, static_cast(SamplesToRemove), EAllowShrinking::No); - BufferedPCMData.Append(PCMData); - } - } - else - { - BufferedPCMData.Append(PCMData); - } -} - -void USTTProcessorWhisper::OnSpeechStateChanged(ESTTTalkingState TalkingState) -{ - CurrentTalkingState = TalkingState; - - if (TalkingState == ESTTTalkingState::BLOCKED) + if (ChainState == ESTTChainState::Discarding) { ClearSTTProcessor(); return; } - if (TalkingState == ESTTTalkingState::SILENCE || TalkingState == ESTTTalkingState::TRANSCRIBING) - { + BufferedPCMData.Append(PCMData); + + if (ChainState == ESTTChainState::Finalizing) StartTranscriptionFromBuffer(); - } } void USTTProcessorWhisper::StartTranscriptionFromBuffer() { - - if (BufferedPCMData.Num() == 0 || !bHasBufferedAudioInformation) + if (BufferedPCMData.Num() == 0) return; TArray PCMDataCopy = BufferedPCMData; FAudioInformation AudioInfoCopy = BufferedAudioInformation; BufferedPCMData.Empty(); - bHasBufferedAudioInformation = false; // Require at least x seconds of audio before sending to Whisper if (AudioInfoCopy.SampleRate > 0 && AudioInfoCopy.NumChannels > 0) @@ -205,7 +137,6 @@ void USTTProcessorWhisper::StartTranscriptionFromBuffer() } STTManager->UserSpeechStateChanged(ESTTTalkingState::TRANSCRIBING); - SendWhisperRequest(MoveTemp(WavData)); } @@ -342,7 +273,6 @@ void USTTProcessorWhisper::PerformHealthCheck() Manager->OnSTTError.Broadcast(TEXT("Whisper initialization check failed: API key invalid (401).")); return; } - Manager->OnSTTFullyInitialized(); }); @@ -444,8 +374,6 @@ void USTTProcessorWhisper::SendWhisperRequest(TArray&& WavData) return; } - UE_LOG(LogTemp, Warning, TEXT("OpenAI says: %s"), *JsonString); - FString Language; if (RootObject->TryGetStringField(TEXT("language"), Language) && !Language.IsEmpty()) { diff --git a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Recorder/STTRecorderAudioData.cpp b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Recorder/STTRecorderAudioData.cpp index 30fa693..c6a9700 100644 --- a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Recorder/STTRecorderAudioData.cpp +++ b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Recorder/STTRecorderAudioData.cpp @@ -25,5 +25,5 @@ void USTTRecorderAudioData::SendAudioDataToSTTModule(TArray InputPCM, int AudioInformation.NumChannels = NumChannels; AudioInformation.SampleRate = InputSampleRate; - OnChunkReceived.ExecuteIfBound(PCMData, AudioInformation); + OnChunkReceived.ExecuteIfBound(PCMData, AudioInformation, ESTTChainState::Processing); } \ No newline at end of file diff --git a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Recorder/STTRecorderDebugFile.cpp b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Recorder/STTRecorderDebugFile.cpp index ee78b56..c1c7b31 100644 --- a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Recorder/STTRecorderDebugFile.cpp +++ b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Recorder/STTRecorderDebugFile.cpp @@ -154,7 +154,7 @@ void USTTRecorderDebugFile::DebugPlayAudioFile(FString FilePath) GetWorld()->GetTimerManager().ClearTimer(PlaybackTimerHandle); } - STTManager->OnSTTFakeButtonStateChanged.Broadcast(true); + STTManager->OnSTTButtonStateChanged.Broadcast(true); GetWorld()->GetTimerManager().SetTimer( PlaybackTimerHandle, // handle to cancel timer at a later time this, // the owning object @@ -167,7 +167,7 @@ void USTTRecorderDebugFile::DebugPlayAudioFile(FString FilePath) void USTTRecorderDebugFile::DebugClearAudioFile() { - STTManager->OnSTTFakeButtonStateChanged.Broadcast(false); + STTManager->OnSTTButtonStateChanged.Broadcast(false); AudioComponent->Stop(); GetWorld()->GetTimerManager().ClearTimer(PlaybackTimerHandle); } @@ -180,13 +180,13 @@ void USTTRecorderDebugFile::SendChunk() if (RemainingSamples <= 0) { GetWorld()->GetTimerManager().ClearTimer(PlaybackTimerHandle); - STTManager->OnSTTFakeButtonStateChanged.Broadcast(false); + STTManager->OnSTTButtonStateChanged.Broadcast(false); STTManager->OnSTTLog.Broadcast(FString::Printf(TEXT("Finished sending %i audio chunks (SampleRate: %i Channels: %i)"), PCMData.Num(), AudioInformation.SampleRate, AudioInformation.NumChannels)); return; } TArray Chunk; Chunk.Append(&PCMData[SentChunks], RemainingSamples); - OnChunkReceived.ExecuteIfBound(Chunk, AudioInformation); + OnChunkReceived.ExecuteIfBound(Chunk, AudioInformation, ESTTChainState::Processing); SentChunks += RemainingSamples; } diff --git a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Recorder/STTRecorderMicrophone.cpp b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Recorder/STTRecorderMicrophone.cpp index 925d95b..a914758 100644 --- a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Recorder/STTRecorderMicrophone.cpp +++ b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Recorder/STTRecorderMicrophone.cpp @@ -297,7 +297,7 @@ void USTTRecorderMicrophone::ProcessChunk() while (AudioQueue.Dequeue(PCMData)) { HasReceivedAudioData = true; - OnChunkReceived.ExecuteIfBound(PCMData, CaptureAudioInformation); + OnChunkReceived.ExecuteIfBound(PCMData, CaptureAudioInformation, ESTTChainState::Processing); ProcessedChunks++; } } diff --git a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Recorder/STTRecorderPrimaryMicrophone.cpp b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Recorder/STTRecorderPrimaryMicrophone.cpp index 7f79c19..18fcb51 100644 --- a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Recorder/STTRecorderPrimaryMicrophone.cpp +++ b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Recorder/STTRecorderPrimaryMicrophone.cpp @@ -304,7 +304,7 @@ void USTTRecorderPrimaryMicrophone::ProcessChunk() //Game Thead Processing HasReceivedAudioData = true; // Write audio data immediately to the PushAudioInputStream - OnChunkReceived.ExecuteIfBound(PCMData, CaptureAudioInformation); + OnChunkReceived.ExecuteIfBound(PCMData, CaptureAudioInformation, ESTTChainState::Processing); ProcessedChunks++; } } diff --git a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Recorder/STTRecorderUnrealMicrophone.cpp b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Recorder/STTRecorderUnrealMicrophone.cpp index c263726..587d605 100644 --- a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Recorder/STTRecorderUnrealMicrophone.cpp +++ b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Recorder/STTRecorderUnrealMicrophone.cpp @@ -304,7 +304,7 @@ void USTTRecorderUnrealMicrophone::ProcessChunk() //Game Thead Processing HasReceivedAudioData = true; // Write audio data immediately to the PushAudioInputStream - OnChunkReceived.ExecuteIfBound(PCMData, CaptureAudioInformation); + OnChunkReceived.ExecuteIfBound(PCMData, CaptureAudioInformation, ESTTChainState::Processing); ProcessedChunks++; } } diff --git a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/STTManagerBase.cpp b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/STTManagerBase.cpp index 3901500..384dd3a 100644 --- a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/STTManagerBase.cpp +++ b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/STTManagerBase.cpp @@ -148,7 +148,6 @@ void USTTManagerBase::ClearSTTManager() void USTTManagerBase::SetBlocked(bool isBlocked) { - if (!bIsInitialized) return; @@ -176,6 +175,13 @@ bool USTTManagerBase::IsBlocked() return (CurrentSpeechState==ESTTTalkingState::BLOCKED); } +void USTTManagerBase::SetLanguage(TArray NewLanguages) +{ + if (!IsValid(ProcessorConfig)) + return; + ProcessorConfig->BaseSettings.STTLanguages = NewLanguages; +} + void USTTManagerBase::AddSpecialWord(FString NewWord) { if (bDebugMode) @@ -261,7 +267,10 @@ void USTTManagerBase::PTTStateChanged(bool BtnPressed) ClearPTTPostRollTimer(); if (BtnPressed) + { + OnSTTButtonStateChanged.Broadcast(true); USTTManagerBase::UserSpeechStateChanged(ESTTTalkingState::TALKING); + } else { if (PTTPostRollTime > 0) { @@ -279,6 +288,7 @@ void USTTManagerBase::PTTStateChanged(bool BtnPressed) void USTTManagerBase::PTTRelease() { + OnSTTButtonStateChanged.Broadcast(false); if(STTProcessor->IsTranscriptionRunning()) USTTManagerBase::UserSpeechStateChanged(ESTTTalkingState::TRANSCRIBING); else diff --git a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Preprocessor/STTPreprocessorBase.h b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Preprocessor/STTPreprocessorBase.h index 22cfce1..6c35df9 100644 --- a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Preprocessor/STTPreprocessorBase.h +++ b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Preprocessor/STTPreprocessorBase.h @@ -10,7 +10,7 @@ class USTTManagerBase; -DECLARE_DELEGATE_TwoParams(FDelegateProcessedChunk, TArray, FAudioInformation); +DECLARE_DELEGATE_ThreeParams(FDelegateProcessedChunk, TArray, FAudioInformation, ESTTChainState); /** * This module processes the audio chunks for the final processor. For example buffer the chunks or change it to 16.000 hz mono audio. @@ -31,7 +31,7 @@ public: virtual void DestroySTTPreprocessor() {}; UFUNCTION() - virtual void OnChunkReceived(TArray PCMData, FAudioInformation AudioInformation) {}; + virtual void OnChunkReceived(TArray PCMData, FAudioInformation AudioInformation, ESTTChainState ChainState) {}; protected: diff --git a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Preprocessor/STTPreprocessorBuffer.h b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Preprocessor/STTPreprocessorBuffer.h index 27de9f0..6e82fb3 100644 --- a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Preprocessor/STTPreprocessorBuffer.h +++ b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Preprocessor/STTPreprocessorBuffer.h @@ -7,7 +7,7 @@ #include "STTPreprocessorBuffer.generated.h" /** - * + * */ UCLASS(Blueprintable, BlueprintType) class AVATARCORE_STT_API USTTPreprocessorBuffer : public USTTPreprocessorBase @@ -18,20 +18,21 @@ class AVATARCORE_STT_API USTTPreprocessorBuffer : public USTTPreprocessorBase int32 BufferSize = 10; //Buffersize in ms UPROPERTY(EditAnywhere) bool CanOverflow = false; //If false Buffersize will be exact otherwise just "send" data when Buffersize is reached. - UPROPERTY(EditAnywhere) - bool FlushOnSilence = false; - void InitSTTPreprocessor(USTTManagerBase* BaseSTTManager, FSTTBaseSettings InSTTBaseSettings, bool InDebugMode = false) override; + // When Finalizing arrives but no full-size Processing chunk was ever dispatched this utterance, + // send Discarding instead — prevents very short accidental utterances from reaching the processor. + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreSTT", meta = (AllowPrivateAccess = "true")) + bool bDiscardWhenNotFilledFullyOnce = false; - void OnChunkReceived(TArray PCMData, FAudioInformation AudioInformation) override; + void InitSTTPreprocessor(USTTManagerBase* BaseSTTManager, FSTTBaseSettings InSTTBaseSettings, bool InDebugMode = false) override; - UFUNCTION() - void OnSpeechChanged(ESTTTalkingState NewSpeechState); + void OnChunkReceived(TArray PCMData, FAudioInformation AudioInformation, ESTTChainState ChainState) override; private: uint32 GetSampleCount(FAudioInformation AudioInformation); TArray Buffer; FAudioInformation BufferAudioInformation; - + bool bHasFilledFully = false; + }; diff --git a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Preprocessor/STTPreprocessorConverter.h b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Preprocessor/STTPreprocessorConverter.h index 4fa4554..4654f8d 100644 --- a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Preprocessor/STTPreprocessorConverter.h +++ b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Preprocessor/STTPreprocessorConverter.h @@ -14,7 +14,7 @@ class AVATARCORE_STT_API USTTPreprocessorConverter : public USTTPreprocessorBase { GENERATED_BODY() - void OnChunkReceived(TArray PCMData, FAudioInformation AudioInformation) override; + void OnChunkReceived(TArray PCMData, FAudioInformation AudioInformation, ESTTChainState ChainState) override; public: diff --git a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Preprocessor/STTPreprocessorDebugger.h b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Preprocessor/STTPreprocessorDebugger.h index 4233808..4cd719c 100644 --- a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Preprocessor/STTPreprocessorDebugger.h +++ b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Preprocessor/STTPreprocessorDebugger.h @@ -19,7 +19,7 @@ class AVATARCORE_STT_API USTTPreprocessorDebugger : public USTTPreprocessorBase void InitSTTPreprocessor(USTTManagerBase* BaseSTTManager, FSTTBaseSettings InSTTBaseSettings, bool InDebugMode = false) override; - void OnChunkReceived(TArray PCMData, FAudioInformation AudioInformation) override; + void OnChunkReceived(TArray PCMData, FAudioInformation AudioInformation, ESTTChainState ChainState) override; void DestroySTTPreprocessor() override; diff --git a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Preprocessor/STTPreprocessorPTT.h b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Preprocessor/STTPreprocessorPTT.h index 1b1dd3c..536f05c 100644 --- a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Preprocessor/STTPreprocessorPTT.h +++ b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Preprocessor/STTPreprocessorPTT.h @@ -18,13 +18,16 @@ class AVATARCORE_STT_API USTTPreprocessorPTT : public USTTPreprocessorBase void DestroySTTPreprocessor() override; - void OnChunkReceived(TArray PCMData, FAudioInformation AudioInformation) override; + void OnChunkReceived(TArray PCMData, FAudioInformation AudioInformation, ESTTChainState ChainState) override; UFUNCTION() void OnUserSpeechStateChanged(ESTTTalkingState NewSpeechState); + UFUNCTION() + void OnPTTStateChanged(bool IsPressed); private: - ESTTTalkingState NewSpeechState = ESTTTalkingState::BLOCKED; + bool PTTPressed = false; + FAudioInformation LastAudioInformation; }; diff --git a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Preprocessor/STTPreprocessorSpeexDSP.h b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Preprocessor/STTPreprocessorSpeexDSP.h index cc252d0..d38ac93 100644 --- a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Preprocessor/STTPreprocessorSpeexDSP.h +++ b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Preprocessor/STTPreprocessorSpeexDSP.h @@ -22,7 +22,7 @@ class AVATARCORE_STT_API USTTPreprocessorSpeexDSP : public USTTPreprocessorBase void PostInitProperties() override; - void OnChunkReceived(TArray PCMData, FAudioInformation AudioInformation) override; + void OnChunkReceived(TArray PCMData, FAudioInformation AudioInformation, ESTTChainState ChainState) override; UFUNCTION(BlueprintCallable, Category = STTManager) void UpdateSpeexDSPSettings(FSpeexDSPSettings InSpeexDSPSettings); diff --git a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Preprocessor/STTPreprocessorVAD.h b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Preprocessor/STTPreprocessorVAD.h index 379baa1..56b497b 100644 --- a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Preprocessor/STTPreprocessorVAD.h +++ b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Preprocessor/STTPreprocessorVAD.h @@ -29,7 +29,7 @@ class AVATARCORE_STT_API USTTPreprocessorVAD : public USTTPreprocessorBase // Vo void DestroySTTPreprocessor() override; - void OnChunkReceived(TArray PCMData, FAudioInformation AudioInformation) override; + void OnChunkReceived(TArray PCMData, FAudioInformation AudioInformation, ESTTChainState ChainState) override; public: @@ -44,4 +44,5 @@ private: int32 lastVADState = -1; float timeInStateInSeconds = 0; + FAudioInformation LastAudioInformation; }; diff --git a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Preprocessor/STTPreprocessorWebRTC.h b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Preprocessor/STTPreprocessorWebRTC.h index 0536c54..e0f1547 100644 --- a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Preprocessor/STTPreprocessorWebRTC.h +++ b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Preprocessor/STTPreprocessorWebRTC.h @@ -24,7 +24,8 @@ public: virtual void DestroySTTPreprocessor() override; virtual void PostInitProperties() override; virtual void OnChunkReceived(TArray PCMData, - FAudioInformation AudioInformation) override; + FAudioInformation AudioInformation, + ESTTChainState ChainState) override; UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "STT|WebRTC") int32 WebRTCStreamDelayMs = 50; UFUNCTION(BlueprintCallable, Category = "STT|WebRTC") diff --git a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Processor/Azure/STTAzureProcessorConfig.h b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Processor/Azure/STTAzureProcessorConfig.h index 7e69364..9a5aa61 100644 --- a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Processor/Azure/STTAzureProcessorConfig.h +++ b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Processor/Azure/STTAzureProcessorConfig.h @@ -6,27 +6,8 @@ #include "Processor/STTBaseProcessorConfig.h" #include "STTAzureProcessorConfig.generated.h" -UENUM(BlueprintType) -enum class EAzureLanguages : uint8 -{ - German_Germany UMETA(DisplayName = "German (Germany)"), - English_UK UMETA(DisplayName = "English (United Kingdom)"), - English_India UMETA(DisplayName = "English (India)"), - English_US UMETA(DisplayName = "English (United States)"), - Spanish_Spain UMETA(DisplayName = "Spanish (Spain)"), - Spanish_Mexico UMETA(DisplayName = "Spanish (Mexico)"), - French_France UMETA(DisplayName = "French (France)"), - Hindi_India UMETA(DisplayName = "Hindi (India)"), - Italian_Italy UMETA(DisplayName = "Italian (Italy)"), - Japanese_Japan UMETA(DisplayName = "Japanese (Japan)"), - Korean_Korea UMETA(DisplayName = "Korean (Korea)"), - Portuguese_Brazil UMETA(DisplayName = "Portuguese (Brazil)"), - Chinese_Simplified UMETA(DisplayName = "Chinese (Mandarin, Simplified)"), - MAX UMETA(Hidden) // Helper for array size checks -}; - /** - * + * */ UCLASS(Blueprintable, BlueprintType) class AVATARCORE_STT_API USTTAzureProcessorConfig : public USTTBaseProcessorConfig @@ -41,6 +22,4 @@ public: FString AzureAPIKey = ""; UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreSTT|Azure", meta = (ExposeOnSpawn = "true")) FString AzureRegion = "germanywestcentral"; - UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreSTT|Azure", meta = (ExposeOnSpawn = "true")) - TArray AzureLanguages = { EAzureLanguages::English_US, EAzureLanguages::German_Germany }; }; diff --git a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Processor/Azure/STTProcessorAzure.h b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Processor/Azure/STTProcessorAzure.h index ff4c068..2b7c806 100644 --- a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Processor/Azure/STTProcessorAzure.h +++ b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Processor/Azure/STTProcessorAzure.h @@ -27,21 +27,19 @@ class AVATARCORE_STT_API USTTProcessorAzure : public USTTProcessorBase void ClearSTTProcessor(); void DestroySTTProcessor() override; - void OnChunkReceived(TArray PCMData, FAudioInformation AudioInformation) override; - + void OnChunkReceived(TArray PCMData, FAudioInformation AudioInformation, ESTTChainState ChainState) override; + public: UFUNCTION(BlueprintCallable, Category = STTManager) - virtual void ChangeAzureLanguage(TArray< EAzureLanguages> InAzureLanguages); - - UFUNCTION() - void OnSpeechStateChanged(ESTTTalkingState TalkingState); + virtual void ChangeAzureLanguage(TArray InLanguages); private: std::shared_ptr config; std::shared_ptr audioConfig; + ESTTChainState LastChainState = ESTTChainState::Discarding; FString intermediateResult = ""; private: @@ -64,7 +62,7 @@ public: void OnAzureError(FString Error); UFUNCTION(BlueprintPure, Category = STTManager) - FString AzureEnumToString(EAzureLanguages Language); + FString AzureEnumToString(ESTTLanguage Language); USTTAzureProcessorConfig* AzureProcessorConfig; }; diff --git a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Processor/Parakeet/STTParakeetProcessorBase.h b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Processor/Parakeet/STTParakeetProcessorBase.h index c9709cc..f1b67b7 100644 --- a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Processor/Parakeet/STTParakeetProcessorBase.h +++ b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Processor/Parakeet/STTParakeetProcessorBase.h @@ -27,7 +27,7 @@ public: void ClearSTTProcessor() override; void DestroySTTProcessor() override; - void OnChunkReceived(TArray PCMData, FAudioInformation AudioInformation) override; + void OnChunkReceived(TArray PCMData, FAudioInformation AudioInformation, ESTTChainState ChainState) override; // Callbacks from FParakeetRunnable (called on GameThread via AsyncTask) void OnParakeetReady(); @@ -35,9 +35,6 @@ public: void OnParakeetFinal(const FString& Text, const FString& Language); void OnParakeetError(const FString& Error); - UFUNCTION() - void OnSpeechStateChanged(ESTTTalkingState TalkingState); - private: USTTParakeetProcessorConfig* ParakeetConfig = nullptr; @@ -50,7 +47,7 @@ private: FTimerHandle ConnectTimerHandle; TSharedPtr ParakeetAddr; int32 ConnectAttempts = 0; - bool bIsEditor = false; + bool bKeepAlive = false; // Process management FString ParakeetBatPath; diff --git a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Processor/Parakeet/STTParakeetProcessorConfig.h b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Processor/Parakeet/STTParakeetProcessorConfig.h index 7d11e04..7dd6594 100644 --- a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Processor/Parakeet/STTParakeetProcessorConfig.h +++ b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Processor/Parakeet/STTParakeetProcessorConfig.h @@ -30,6 +30,9 @@ public: UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreSTT|Parakeet", meta = (ExposeOnSpawn = "true")) int32 Port = 40200; + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreSTT|Parakeet", meta = (ExposeOnSpawn = "true")) + ESTTKeepAliveRule KeepAliveRule = ESTTKeepAliveRule::EditorOnly; + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreSTT|Parakeet", meta = (ExposeOnSpawn = "true")) FString Device = "cuda:0"; diff --git a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Processor/RealtimeAPI/STTProcessorRealtimeAPI.h b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Processor/RealtimeAPI/STTProcessorRealtimeAPI.h index 967e07d..be54d98 100644 --- a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Processor/RealtimeAPI/STTProcessorRealtimeAPI.h +++ b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Processor/RealtimeAPI/STTProcessorRealtimeAPI.h @@ -24,7 +24,7 @@ class AVATARCORE_STT_API USTTProcessorRealtimeAPI : public USTTProcessorBase UFUNCTION(BlueprintCallable, Category = STTManager) void ClearToRealtimeAPI(); - void OnChunkReceived(TArray PCMData, FAudioInformation AudioInformation) override; + void OnChunkReceived(TArray PCMData, FAudioInformation AudioInformation, ESTTChainState ChainState) override; private: diff --git a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Processor/STTBaseProcessorConfig.h b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Processor/STTBaseProcessorConfig.h index 4a2414d..4b32940 100644 --- a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Processor/STTBaseProcessorConfig.h +++ b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Processor/STTBaseProcessorConfig.h @@ -7,6 +7,13 @@ #include "STTStructs.h" #include "STTBaseProcessorConfig.generated.h" +UENUM(BlueprintType) +enum class ESTTKeepAliveRule : uint8 { + EditorOnly UMETA(DisplayName = "Only keep STT Module open in Editor Mode"), + Never UMETA(DisplayName = "Never keep STT Module alive."), + Always UMETA(DisplayName = "Keep STT Module alive in Editor and Shipping mode.") +}; + class USTTProcessorBase; /** diff --git a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Processor/STTProcessorBase.h b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Processor/STTProcessorBase.h index 32377e9..bb01cba 100644 --- a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Processor/STTProcessorBase.h +++ b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Processor/STTProcessorBase.h @@ -42,7 +42,7 @@ public: void OnTranscriptionStarted(); UFUNCTION() - virtual void OnChunkReceived(TArray PCMData, FAudioInformation AudioInformation) {}; + virtual void OnChunkReceived(TArray PCMData, FAudioInformation AudioInformation, ESTTChainState ChainState) {}; protected: diff --git a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Processor/STTProcessorDebugSaveWav.h b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Processor/STTProcessorDebugSaveWav.h index aacd279..3e8ebba 100644 --- a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Processor/STTProcessorDebugSaveWav.h +++ b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Processor/STTProcessorDebugSaveWav.h @@ -16,7 +16,7 @@ class AVATARCORE_STT_API USTTProcessorDebugSaveWav : public USTTProcessorBase void DestroySTTProcessor() override; - void OnChunkReceived(TArray PCMData, FAudioInformation AudioInformation) override; + void OnChunkReceived(TArray PCMData, FAudioInformation AudioInformation, ESTTChainState ChainState) override; private: diff --git a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Processor/Whisper/STTProcessorWhisper.h b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Processor/Whisper/STTProcessorWhisper.h index 2010520..c4b7e8a 100644 --- a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Processor/Whisper/STTProcessorWhisper.h +++ b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Processor/Whisper/STTProcessorWhisper.h @@ -21,10 +21,7 @@ public: virtual void ClearSTTProcessor() override; virtual void DestroySTTProcessor() override; - virtual void OnChunkReceived(TArray PCMData, FAudioInformation AudioInformation) override; - - UFUNCTION() - void OnSpeechStateChanged(ESTTTalkingState TalkingState); + virtual void OnChunkReceived(TArray PCMData, FAudioInformation AudioInformation, ESTTChainState ChainState) override; private: @@ -41,7 +38,5 @@ private: FString NormalizedWhisperURL; TArray BufferedPCMData; FAudioInformation BufferedAudioInformation; - bool bHasBufferedAudioInformation = false; TArray> ActiveRequests; - ESTTTalkingState CurrentTalkingState = ESTTTalkingState::SILENCE; }; diff --git a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Recorder/STTRecorderBase.h b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Recorder/STTRecorderBase.h index 750f212..21301aa 100644 --- a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Recorder/STTRecorderBase.h +++ b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Recorder/STTRecorderBase.h @@ -9,7 +9,7 @@ class USTTManagerBase; -DECLARE_DELEGATE_TwoParams(FDelegateUnprocessedChunkReceived, TArray, FAudioInformation); +DECLARE_DELEGATE_ThreeParams(FDelegateUnprocessedChunkReceived, TArray, FAudioInformation, ESTTChainState); /** * This module is the producer of the audio chunks (might be a microphone or a webstream (in the case of pixel streaming). diff --git a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/STTManagerBase.h b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/STTManagerBase.h index cdf076b..99a2772 100644 --- a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/STTManagerBase.h +++ b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/STTManagerBase.h @@ -17,7 +17,7 @@ DECLARE_DYNAMIC_MULTICAST_DELEGATE_ThreeParams(FMulticastDelegateTranscriptionRe DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FMulticastDelegateSpeechStateChanged, ESTTTalkingState, TalkingState); DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FMulticastDelegateSpeechStateChangedForUI, ESTTTalkingState, TalkingState); DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FMulticastDelegateSTTBlocked, bool, IsBlocked); -DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FMulticastDelegateFakeButtonStateChanged, bool, IsPressed); +DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FMulticastDelegateButtonStateChanged, bool, IsPressed); DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FMulticastDelegateSTTLog, FString, LogContent); DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FMulticastDelegateSTTError, FString, LogError); DECLARE_DYNAMIC_MULTICAST_DELEGATE(FMulticastDelegateSpeechDetectedWhileBlocked); @@ -44,7 +44,7 @@ public: UPROPERTY(BlueprintAssignable, Category = "AvatarCoreSTT") FMulticastDelegateTranscriptionReceived OnTranscriptionReceived; UPROPERTY(BlueprintAssignable, Category = "AvatarCoreSTT") - FMulticastDelegateFakeButtonStateChanged OnSTTFakeButtonStateChanged; + FMulticastDelegateButtonStateChanged OnSTTButtonStateChanged; UPROPERTY(BlueprintAssignable, Category = "AvatarCoreSTT") FMulticastDelegateSTTLog OnSTTLog; UPROPERTY(BlueprintAssignable, Category = "AvatarCoreSTT") @@ -98,6 +98,9 @@ public: UFUNCTION(BlueprintPure, Category = "AvatarCoreSTT") bool IsBlocked(); + UFUNCTION(BlueprintCallable, Category = "AvatarCoreSTT") + void SetLanguage(TArray NewLanguages); + UFUNCTION(BlueprintCallable, Category = "AvatarCoreSTT") void AddSpecialWord(FString NewWord); diff --git a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/STTStructs.h b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/STTStructs.h index 29ed520..f7cc50e 100644 --- a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/STTStructs.h +++ b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/STTStructs.h @@ -23,6 +23,14 @@ struct FAudioInformation int32 SampleRate = 16000; // 16kHz }; +UENUM(BlueprintType) +enum class ESTTChainState : uint8 +{ + Processing UMETA(DisplayName = "Processing"), // normal audio chunk + Finalizing UMETA(DisplayName = "Finalizing"), // end of utterance - trigger transcription + Discarding UMETA(DisplayName = "Discarding"), // BLOCKED/abort - clear everything +}; + UENUM(BlueprintType) enum ESTTTalkingState { @@ -35,8 +43,8 @@ enum ESTTTalkingState UENUM(BlueprintType) enum class ESTTTranscriptionType : uint8 { - OpenAI = 0 UMETA(DisplayName = "OpenAI Transcription"), - Azure = 1 UMETA(DisplayName = "Mircosoft Azure Congnitive Speech Services"), + Azure = 0 UMETA(DisplayName = "Mircosoft Azure Congnitive Speech Services"), + OpenAI = 1 UMETA(DisplayName = "OpenAI Transcription"), Parakeet = 2 UMETA(DisplayName = "nvidia NeMo Parakeet (local transcription)"), }; @@ -62,6 +70,54 @@ enum class ESpeexDSPState : uint8 SPEEXPREPROCESS_SET_AGC_TARGET = 46 UMETA(DisplayName = "preprocessor Automatic Gain Control level (int32)") }; +UENUM(BlueprintType) +enum class ESTTLanguage : uint8 +{ + NONE UMETA(DisplayName = "Unset"), + en UMETA(DisplayName = "English"), + fr UMETA(DisplayName = "French"), + de UMETA(DisplayName = "German"), + es UMETA(DisplayName = "Spanish"), + pt UMETA(DisplayName = "Portuguese"), + zh UMETA(DisplayName = "Chinese"), + ja UMETA(DisplayName = "Japanese"), + hi UMETA(DisplayName = "Hindi"), + it UMETA(DisplayName = "Italian"), + ko UMETA(DisplayName = "Korean"), + nl UMETA(DisplayName = "Dutch"), + pl UMETA(DisplayName = "Polish"), + ru UMETA(DisplayName = "Russian"), + sv UMETA(DisplayName = "Swedish"), + tr UMETA(DisplayName = "Turkish"), + tl UMETA(DisplayName = "Filipino"), + bg UMETA(DisplayName = "Bulgarian"), + ro UMETA(DisplayName = "Romanian"), + ar UMETA(DisplayName = "Arabic"), + cs UMETA(DisplayName = "Czech"), + el UMETA(DisplayName = "Greek"), + fi UMETA(DisplayName = "Finnish"), + hr UMETA(DisplayName = "Croatian"), + ms UMETA(DisplayName = "Malay"), + sk UMETA(DisplayName = "Slovak"), + da UMETA(DisplayName = "Danish"), + ta UMETA(DisplayName = "Tamil"), + uk UMETA(DisplayName = "Ukrainian"), + hu UMETA(DisplayName = "Hungarian"), + no UMETA(DisplayName = "Norwegian"), + vi UMETA(DisplayName = "Vietnamese"), + bn UMETA(DisplayName = "Bengali"), + th UMETA(DisplayName = "Thai"), + he UMETA(DisplayName = "Hebrew"), + ka UMETA(DisplayName = "Georgian"), + id UMETA(DisplayName = "Indonesian"), + te UMETA(DisplayName = "Telugu"), + gu UMETA(DisplayName = "Gujarati"), + kn UMETA(DisplayName = "Kannada"), + ml UMETA(DisplayName = "Malayalam"), + mr UMETA(DisplayName = "Marathi"), + pa UMETA(DisplayName = "Punjabi"), +}; + USTRUCT(BlueprintType) struct FWebRTCSettings { @@ -205,6 +261,8 @@ struct FSTTBaseSettings FVADSettings VADSettings; UPROPERTY(EditAnywhere, BlueprintReadWrite, meta = (ToolTip = "Settings of the SpeexDSP Module", Category = "STT|Base")) FSpeexDSPSettings SpeexDSPSettings; + UPROPERTY(EditAnywhere, BlueprintReadWrite, meta = (ToolTip = "All languages the STT module should understand simultaneously.", Category = "STT|Base")) + TArray STTLanguages = { ESTTLanguage::de, ESTTLanguage::en }; UPROPERTY(EditAnywhere, BlueprintReadWrite, meta = (ToolTip = "Transcriptions to always change to another word.", Category = "STT|Base")) TArray STTReplacements; UPROPERTY(EditAnywhere, BlueprintReadWrite, meta = (ToolTip = "Special words that the transcription service needs to know (e.g. b.ReX or Bruce-B).", Category = "STT|Base")) diff --git a/Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Private/Cartesia/CartesiaTTSManager.cpp b/Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Private/Cartesia/CartesiaTTSManager.cpp index aeee923..6753742 100644 --- a/Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Private/Cartesia/CartesiaTTSManager.cpp +++ b/Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Private/Cartesia/CartesiaTTSManager.cpp @@ -23,11 +23,30 @@ namespace FJsonSerializer::Serialize(Obj.ToSharedRef(), Writer); return Out; } + + static const FString TTSLanguageStrings[] = { + TEXT("en"), TEXT("fr"), TEXT("de"), TEXT("es"), TEXT("pt"), TEXT("zh"), TEXT("ja"), + TEXT("hi"), TEXT("it"), TEXT("ko"), TEXT("nl"), TEXT("pl"), TEXT("ru"), TEXT("sv"), + TEXT("tr"), TEXT("tl"), TEXT("bg"), TEXT("ro"), TEXT("ar"), TEXT("cs"), TEXT("el"), + TEXT("fi"), TEXT("hr"), TEXT("ms"), TEXT("sk"), TEXT("da"), TEXT("ta"), TEXT("uk"), + TEXT("hu"), TEXT("no"), TEXT("vi"), TEXT("bn"), TEXT("th"), TEXT("he"), TEXT("ka"), + TEXT("id"), TEXT("te"), TEXT("gu"), TEXT("kn"), TEXT("ml"), TEXT("mr"), TEXT("pa") + }; + + static FString TTSLanguageToString(ETTSLanguage Language) + { + const int32 Index = static_cast(Language); + if (Index >= 0 && Index < UE_ARRAY_COUNT(TTSLanguageStrings)) + { + return TTSLanguageStrings[Index]; + } + return TEXT("de"); + } } FString UCartesiaTTSManager::BuildWebSocketUrl() const { - const FString Base = (CartesiaTTSConfig && !CartesiaTTSConfig->CartesiaBaseURI.IsEmpty()) ? CartesiaTTSConfig->CartesiaBaseURI : TEXT("api.cartesia.ai"); + const FString Base = (CartesiaTTSConfig && !CartesiaTTSConfig->CartesiaTTSSettings.CartesiaBaseURI.IsEmpty()) ? CartesiaTTSConfig->CartesiaTTSSettings.CartesiaBaseURI : TEXT("api.cartesia.ai"); return FString::Printf(TEXT("wss://%s/tts/websocket"), *Base); } @@ -59,7 +78,7 @@ void UCartesiaTTSManager::InitTTSManager(UTTSBaseConfig* InTSSConfig, bool Debug return; } - bSupportsStreamedInput = CartesiaTTSConfig->StreamInputText; + bSupportsStreamedInput = CartesiaTTSConfig->CartesiaTTSSettings.StreamInputText; if (!FModuleManager::Get().IsModuleLoaded("WebSockets")) { @@ -100,6 +119,7 @@ void UCartesiaTTSManager::ClearTTS() void UCartesiaTTSManager::CloseAndRemoveSocket(int32 TaskID, bool bSendFinal) { TSharedPtr Socket; + bool bNoMoreSockets = false; { FScopeLock Lock(&SocketsCS); if (TSharedPtr* Found = ActiveSockets.Find(TaskID)) @@ -110,14 +130,19 @@ void UCartesiaTTSManager::CloseAndRemoveSocket(int32 TaskID, bool bSendFinal) TaskContextIds.Remove(TaskID); TasksFlushSent.Remove(TaskID); TasksSentFirst.Remove(TaskID); + bNoMoreSockets = ActiveSockets.Num() == 0; } if (Socket.IsValid()) { if (bSendFinal) { - AsyncTask(ENamedThreads::GameThread, [this, TaskID]() + AsyncTask(ENamedThreads::GameThread, [this, TaskID, bNoMoreSockets]() { + if (bNoMoreSockets) + { + bReceivedFinalInput = true; + } OnGeneratedAudioChunkReceivedByID(TaskID, TArray(), true); }); } @@ -160,27 +185,31 @@ void UCartesiaTTSManager::SendTranscriptMessage(int32 TaskID, const FString& Tra } TSharedPtr Obj = MakeShared(); - Obj->SetStringField(TEXT("model_id"), CartesiaTTSConfig->CartesiaModelId); + Obj->SetStringField(TEXT("model_id"), CartesiaTTSConfig->CartesiaTTSSettings.CartesiaModelId); Obj->SetStringField(TEXT("transcript"), Transcript); - Obj->SetStringField(TEXT("language"), CartesiaTTSConfig->CartesiaLanguage); + Obj->SetStringField(TEXT("language"), TTSLanguageToString(CartesiaTTSConfig->GlobalTTSSettings.Language)); Obj->SetStringField(TEXT("context_id"), GetOrCreateContextId(TaskID)); Obj->SetBoolField(TEXT("continue"), bContinue); TSharedPtr VoiceObj = MakeShared(); VoiceObj->SetStringField(TEXT("mode"), TEXT("id")); - VoiceObj->SetStringField(TEXT("id"), CartesiaTTSConfig->CartesiaVoiceId); + VoiceObj->SetStringField(TEXT("id"), CartesiaTTSConfig->CartesiaTTSSettings.CartesiaVoiceId); Obj->SetObjectField(TEXT("voice"), VoiceObj); TSharedPtr OutputObj = MakeShared(); - OutputObj->SetStringField(TEXT("container"), CartesiaTTSConfig->CartesiaContainer); - OutputObj->SetStringField(TEXT("encoding"), CartesiaTTSConfig->CartesiaEncoding); - const int32 EffectiveSR = (TTSConfig && TTSConfig->ResampleToSampleRate > 0) ? TTSConfig->ResampleToSampleRate : (TTSConfig ? TTSConfig->AudioSampleRate : 24000); - OutputObj->SetNumberField(TEXT("sample_rate"), EffectiveSR); + OutputObj->SetStringField(TEXT("container"), CartesiaTTSConfig->CartesiaTTSSettings.CartesiaContainer); + OutputObj->SetStringField(TEXT("encoding"), CartesiaTTSConfig->CartesiaTTSSettings.CartesiaEncoding); + OutputObj->SetNumberField(TEXT("sample_rate"), (TTSConfig ? TTSConfig->GlobalTTSSettings.AudioSampleRate : 24000)); Obj->SetObjectField(TEXT("output_format"), OutputObj); Obj->SetBoolField(TEXT("add_timestamps"), false); SendJsonForTask(TaskID, Obj); + + { + FScopeLock Lock(&SocketsCS); + TasksSentFirst.Add(TaskID); + } } void UCartesiaTTSManager::SendFlushMessage(int32 TaskID) @@ -205,23 +234,22 @@ void UCartesiaTTSManager::SendFlushMessage(int32 TaskID) } TSharedPtr Obj = MakeShared(); - Obj->SetStringField(TEXT("model_id"), CartesiaTTSConfig->CartesiaModelId); + Obj->SetStringField(TEXT("model_id"), CartesiaTTSConfig->CartesiaTTSSettings.CartesiaModelId); Obj->SetStringField(TEXT("transcript"), TEXT("")); - Obj->SetStringField(TEXT("language"), CartesiaTTSConfig->CartesiaLanguage); + Obj->SetStringField(TEXT("language"), TTSLanguageToString(CartesiaTTSConfig->GlobalTTSSettings.Language)); Obj->SetStringField(TEXT("context_id"), GetOrCreateContextId(TaskID)); Obj->SetBoolField(TEXT("continue"), true); Obj->SetBoolField(TEXT("flush"), true); TSharedPtr VoiceObj = MakeShared(); VoiceObj->SetStringField(TEXT("mode"), TEXT("id")); - VoiceObj->SetStringField(TEXT("id"), CartesiaTTSConfig->CartesiaVoiceId); + VoiceObj->SetStringField(TEXT("id"), CartesiaTTSConfig->CartesiaTTSSettings.CartesiaVoiceId); Obj->SetObjectField(TEXT("voice"), VoiceObj); TSharedPtr OutputObj = MakeShared(); - OutputObj->SetStringField(TEXT("container"), CartesiaTTSConfig->CartesiaContainer); - OutputObj->SetStringField(TEXT("encoding"), CartesiaTTSConfig->CartesiaEncoding); - const int32 EffectiveSR = (TTSConfig && TTSConfig->ResampleToSampleRate > 0) ? TTSConfig->ResampleToSampleRate : (TTSConfig ? TTSConfig->AudioSampleRate : 24000); - OutputObj->SetNumberField(TEXT("sample_rate"), EffectiveSR); + OutputObj->SetStringField(TEXT("container"), CartesiaTTSConfig->CartesiaTTSSettings.CartesiaContainer); + OutputObj->SetStringField(TEXT("encoding"), CartesiaTTSConfig->CartesiaTTSSettings.CartesiaEncoding); + OutputObj->SetNumberField(TEXT("sample_rate"), (TTSConfig ? TTSConfig->GlobalTTSSettings.AudioSampleRate : 24000)); Obj->SetObjectField(TEXT("output_format"), OutputObj); Obj->SetBoolField(TEXT("add_timestamps"), false); @@ -242,12 +270,12 @@ void UCartesiaTTSManager::StartStreamingGeneration(int32 TaskID, const FString& TTSError(TEXT("Cartesia config not set")); return; } - if (CartesiaTTSConfig->CartesiaAPIKey.IsEmpty()) + if (CartesiaTTSConfig->CartesiaTTSSettings.CartesiaAPIKey.IsEmpty()) { TTSError(TEXT("Cartesia API key is empty")); return; } - if (CartesiaTTSConfig->CartesiaVoiceId.IsEmpty()) + if (CartesiaTTSConfig->CartesiaTTSSettings.CartesiaVoiceId.IsEmpty()) { TTSError(TEXT("Cartesia VoiceId is empty")); return; @@ -256,8 +284,8 @@ void UCartesiaTTSManager::StartStreamingGeneration(int32 TaskID, const FString& const FString Url = BuildWebSocketUrl(); TMap Headers; - Headers.Add(TEXT("X-API-Key"), CartesiaTTSConfig->CartesiaAPIKey); - Headers.Add(TEXT("Cartesia-Version"), CartesiaTTSConfig->CartesiaVersion); + Headers.Add(TEXT("X-API-Key"), CartesiaTTSConfig->CartesiaTTSSettings.CartesiaAPIKey); + Headers.Add(TEXT("Cartesia-Version"), CartesiaTTSConfig->CartesiaTTSSettings.CartesiaVersion); Headers.Add(TEXT("Accept"), TEXT("application/json")); Headers.Add(TEXT("User-Agent"), TEXT("AvatarCoreTTS/1.0")); Headers.Add(TEXT("Content-Type"), TEXT("application/json")); diff --git a/Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Private/Cartesia/TTSCartesiaConfig.cpp b/Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Private/Cartesia/TTSCartesiaConfig.cpp index 033f289..cb4da23 100644 --- a/Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Private/Cartesia/TTSCartesiaConfig.cpp +++ b/Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Private/Cartesia/TTSCartesiaConfig.cpp @@ -12,7 +12,7 @@ UTTSCartesiaConfig::UTTSCartesiaConfig(const FObjectInitializer& ObjectInitializ FString UTTSCartesiaConfig::GetHashPrefix() const { - const int32 EffectiveSR = (ResampleToSampleRate > 0) ? ResampleToSampleRate : AudioSampleRate; - return FString::Printf(TEXT("%d|%d|%s|%s|%s|%s"), AudioNumChannels, EffectiveSR, *CartesiaVoiceId, *CartesiaModelId, *CartesiaContainer, *CartesiaEncoding); + const int32 EffectiveSR = (GlobalTTSSettings.ResampleToSampleRate > 0) ? GlobalTTSSettings.ResampleToSampleRate : GlobalTTSSettings.AudioSampleRate; + return FString::Printf(TEXT("%d|%d|%s|%s|%s|%s"), GlobalTTSSettings.AudioNumChannels, EffectiveSR, *CartesiaTTSSettings.CartesiaVoiceId, *CartesiaTTSSettings.CartesiaModelId, *CartesiaTTSSettings.CartesiaContainer, *CartesiaTTSSettings.CartesiaEncoding); } diff --git a/Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Private/Elevenlabs/ElevenlabsTTSConfig.cpp b/Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Private/Elevenlabs/ElevenlabsTTSConfig.cpp index 624f52e..c1b909c 100644 --- a/Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Private/Elevenlabs/ElevenlabsTTSConfig.cpp +++ b/Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Private/Elevenlabs/ElevenlabsTTSConfig.cpp @@ -11,5 +11,5 @@ UElevenlabsTTSConfig::UElevenlabsTTSConfig(const FObjectInitializer& ObjectIniti FString UElevenlabsTTSConfig::GetHashPrefix() const { - return FString::FromInt(AudioNumChannels) + FString::FromInt(AudioSampleRate) + ElevenlabsVoiceID; + return FString::FromInt(GlobalTTSSettings.AudioNumChannels) + FString::FromInt(GlobalTTSSettings.AudioSampleRate) + ElevenlabsTTSSettings.ElevenlabsVoiceID; } diff --git a/Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Private/Elevenlabs/ElevenlabsTTSManager.cpp b/Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Private/Elevenlabs/ElevenlabsTTSManager.cpp index dbde99e..bb6801b 100644 --- a/Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Private/Elevenlabs/ElevenlabsTTSManager.cpp +++ b/Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Private/Elevenlabs/ElevenlabsTTSManager.cpp @@ -33,7 +33,7 @@ void UElevenlabsTTSManager::InitTTSManager(UTTSBaseConfig* InTSSConfig, bool Deb return; } - bSupportsStreamedInput = ElevenlabsTTSConfig->StreamInputText; + bSupportsStreamedInput = ElevenlabsTTSConfig->ElevenlabsTTSSettings.StreamInputText; // Ensure WebSockets module is loaded if (!FModuleManager::Get().IsModuleLoaded("WebSockets")) @@ -78,7 +78,7 @@ void UElevenlabsTTSManager::ClearTTS() FString UElevenlabsTTSManager::ChooseOutputFormat() const { // Map desired output sample rate (or default) to ElevenLabs output format - const int32 SR = (TTSConfig && TTSConfig->ResampleToSampleRate > 0) ? TTSConfig->ResampleToSampleRate : (TTSConfig ? TTSConfig->AudioSampleRate : 24000); + const int32 SR = (TTSConfig && TTSConfig->GlobalTTSSettings.ResampleToSampleRate > 0) ? TTSConfig->GlobalTTSSettings.ResampleToSampleRate : (TTSConfig ? TTSConfig->GlobalTTSSettings.AudioSampleRate : 24000); switch (SR) { case 16000: return TEXT("pcm_16000"); @@ -123,12 +123,12 @@ void UElevenlabsTTSManager::GenerateAudio(FTTSTask& Task) TTSError(TEXT("ElevenLabs config not set")); return; } - if (ElevenlabsTTSConfig->ElevenlabsAPIKey.IsEmpty()) + if (ElevenlabsTTSConfig->ElevenlabsTTSSettings.ElevenlabsAPIKey.IsEmpty()) { TTSError(TEXT("ElevenLabs API key is empty")); return; } - if (ElevenlabsTTSConfig->ElevenlabsVoiceID.IsEmpty()) + if (ElevenlabsTTSConfig->ElevenlabsTTSSettings.ElevenlabsVoiceID.IsEmpty()) { TTSError(TEXT("ElevenLabs VoiceID is empty")); return; @@ -137,17 +137,17 @@ void UElevenlabsTTSManager::GenerateAudio(FTTSTask& Task) FString OutputFmt = ChooseOutputFormat(); const FString Url = BuildElevenLabsWSUrl( - ElevenlabsTTSConfig->ElevenlabsURI, - ElevenlabsTTSConfig->ElevenlabsVoiceID, + ElevenlabsTTSConfig->ElevenlabsTTSSettings.ElevenlabsURI, + ElevenlabsTTSConfig->ElevenlabsTTSSettings.ElevenlabsVoiceID, OutputFmt, Task.Text.Contains("<"), - bIsPreCaching ? ElevenlabsTTSConfig->ElevenlabsModelID_PreCache : ElevenlabsTTSConfig->ElevenlabsModelID, - ElevenlabsTTSConfig->ElevenlabsAPIKey + bIsPreCaching ? ElevenlabsTTSConfig->ElevenlabsTTSSettings.ElevenlabsModelID_PreCache : ElevenlabsTTSConfig->ElevenlabsTTSSettings.ElevenlabsModelID, + ElevenlabsTTSConfig->ElevenlabsTTSSettings.ElevenlabsAPIKey ); TTSLog(FString::Printf(TEXT("[ElevenLabs] Connecting URL: %s"), *Url)); TMap Headers; - Headers.Add(TEXT("xi-api-key"), ElevenlabsTTSConfig->ElevenlabsAPIKey); + Headers.Add(TEXT("xi-api-key"), ElevenlabsTTSConfig->ElevenlabsTTSSettings.ElevenlabsAPIKey); Headers.Add(TEXT("Accept"), TEXT("application/json")); Headers.Add(TEXT("User-Agent"), TEXT("AvatarCoreTTS/1.0")); Headers.Add(TEXT("Content-Type"), TEXT("application/json")); @@ -278,12 +278,12 @@ void UElevenlabsTTSManager::StartStreamingGeneration(int32 TaskID, const FString TTSError(TEXT("ElevenLabs config not set")); return; } - if (ElevenlabsTTSConfig->ElevenlabsAPIKey.IsEmpty()) + if (ElevenlabsTTSConfig->ElevenlabsTTSSettings.ElevenlabsAPIKey.IsEmpty()) { TTSError(TEXT("ElevenLabs API key is empty")); return; } - if (ElevenlabsTTSConfig->ElevenlabsVoiceID.IsEmpty()) + if (ElevenlabsTTSConfig->ElevenlabsTTSSettings.ElevenlabsVoiceID.IsEmpty()) { TTSError(TEXT("ElevenLabs VoiceID is empty")); return; @@ -291,17 +291,17 @@ void UElevenlabsTTSManager::StartStreamingGeneration(int32 TaskID, const FString const FString OutputFmt = ChooseOutputFormat(); const FString Url = BuildElevenLabsWSUrl( - ElevenlabsTTSConfig->ElevenlabsURI, - ElevenlabsTTSConfig->ElevenlabsVoiceID, + ElevenlabsTTSConfig->ElevenlabsTTSSettings.ElevenlabsURI, + ElevenlabsTTSConfig->ElevenlabsTTSSettings.ElevenlabsVoiceID, OutputFmt, false, - ElevenlabsTTSConfig->ElevenlabsModelID, - ElevenlabsTTSConfig->ElevenlabsAPIKey + ElevenlabsTTSConfig->ElevenlabsTTSSettings.ElevenlabsModelID, + ElevenlabsTTSConfig->ElevenlabsTTSSettings.ElevenlabsAPIKey ); TTSLog(FString::Printf(TEXT("[ElevenLabs] Connecting URL (streaming): %s"), *Url)); TMap Headers; - Headers.Add(TEXT("xi-api-key"), ElevenlabsTTSConfig->ElevenlabsAPIKey); + Headers.Add(TEXT("xi-api-key"), ElevenlabsTTSConfig->ElevenlabsTTSSettings.ElevenlabsAPIKey); Headers.Add(TEXT("Accept"), TEXT("application/json")); Headers.Add(TEXT("User-Agent"), TEXT("AvatarCoreTTS/1.0")); Headers.Add(TEXT("Content-Type"), TEXT("application/json")); diff --git a/Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Private/RealtimeAPI/TTSRealtimeAPIConfig.cpp b/Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Private/RealtimeAPI/TTSRealtimeAPIConfig.cpp index 02deb89..b57ad3b 100644 --- a/Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Private/RealtimeAPI/TTSRealtimeAPIConfig.cpp +++ b/Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Private/RealtimeAPI/TTSRealtimeAPIConfig.cpp @@ -6,7 +6,7 @@ UTTSRealtimeAPIConfig::UTTSRealtimeAPIConfig(const FObjectInitializer& ObjectInitializer) { TSSManagerClass = URealtimeAPI_TTSManager::StaticClass(); - bCommaSplitRule = ECommaSplitRule::NeverSplit; + GlobalTTSSettings.bCommaSplitRule = ECommaSplitRule::NeverSplit; } FString UTTSRealtimeAPIConfig::GetHashPrefix() const diff --git a/Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Private/TTSManagerBase.cpp b/Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Private/TTSManagerBase.cpp index faccf5f..d3727f1 100644 --- a/Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Private/TTSManagerBase.cpp +++ b/Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Private/TTSManagerBase.cpp @@ -53,7 +53,7 @@ void UTTSManagerBase::DeinitTTSManager() void UTTSManagerBase::SetMaxConcurrentGenerations(int32 NewMaxConcurrentGenerations) { TTSLog(FString::Printf(TEXT("Changed Max concurrent generations to: '%i'"), NewMaxConcurrentGenerations)); - TTSConfig->MaxConcurrentGenerations = NewMaxConcurrentGenerations; + TTSConfig->GlobalTTSSettings.MaxConcurrentGenerations = NewMaxConcurrentGenerations; } void UTTSManagerBase::TTSLog(const FString& LogMessage, bool AlwaysShow) @@ -203,9 +203,9 @@ FString UTTSManagerBase::ProcessTextForGeneration(const FString& Input) Result = Result.TrimStartAndEnd(); // Apply word/phrase replacements from config (case-insensitive) - if (TTSConfig && TTSConfig->WordReplacements.Num() > 0) + if (TTSConfig && TTSConfig->GlobalTTSSettings.WordReplacements.Num() > 0) { - for (const TPair& Pair : TTSConfig->WordReplacements) + for (const TPair& Pair : TTSConfig->GlobalTTSSettings.WordReplacements) { const FString& From = Pair.Key; const FString& To = Pair.Value; @@ -369,7 +369,13 @@ void UTTSManagerBase::ClearTTS() void UTTSManagerBase::SetCachingEnabled(bool bEnabled) { - TTSConfig->UseCacheSystem = bEnabled; + TTSConfig->GlobalTTSSettings.UseCacheSystem = bEnabled; +} + +void UTTSManagerBase::SetLanguage(ETTSLanguage NewLanguage) +{ + if (!TTSConfig) return; + TTSConfig->GlobalTTSSettings.Language = NewLanguage; } bool UTTSManagerBase::IsTTSBusy() @@ -413,7 +419,7 @@ void UTTSManagerBase::AddTextChunk(const FString& TextChunk, bool IsFinal) if (StreamingTaskID == -1) { const int32 CurrentlyGenerating = GetCurrentlyGeneratingTaskCount_NoLock(); - if (CurrentlyGenerating < TTSConfig->MaxConcurrentGenerations) + if (CurrentlyGenerating < TTSConfig->GlobalTTSSettings.MaxConcurrentGenerations) { FTTSTask NewTask; NewTask.TaskID = NextTaskID++; @@ -524,11 +530,11 @@ void UTTSManagerBase::OnGeneratedAudioChunkReceived(FTTSTask& Task, const TArray { TArray ProcessedAudio; - if (TTSConfig->ResampleToSampleRate > 0 && TTSConfig->ResampleToSampleRate != TTSConfig->AudioSampleRate) + if (TTSConfig->GlobalTTSSettings.ResampleToSampleRate > 0 && TTSConfig->GlobalTTSSettings.ResampleToSampleRate != TTSConfig->GlobalTTSSettings.AudioSampleRate) { ProcessedAudio = ResampleAudio(AudioData); TTSLog(FString::Printf(TEXT("Resampled audio from %d Hz to %d Hz"), - TTSConfig->AudioSampleRate, TTSConfig->ResampleToSampleRate)); + TTSConfig->GlobalTTSSettings.AudioSampleRate, TTSConfig->GlobalTTSSettings.ResampleToSampleRate)); } else { @@ -584,8 +590,15 @@ void UTTSManagerBase::OnGeneratedAudioChunkReceived(FTTSTask& Task, const TArray } TargetTaskPtr->bIsGenerating = !IsLastChunk; + // If all audio has already been rendered by the time generation ends, mark complete now. + // Without this, bIsComplete is only set in ConsumeRenderedBytes — but if the audio component + // already stopped (WaitingForChunks underrun), ConsumeRenderedBytes will never run again. + if (IsLastChunk && !TargetTaskPtr->bIsComplete && TargetTaskPtr->PlaybackCursor >= TargetTaskPtr->AudioData.Num()) + { + TargetTaskPtr->bIsComplete = true; + } } - if (IsLastChunk && TTSConfig->UseCacheSystem && !bIsPreCaching) + if (IsLastChunk && TTSConfig->GlobalTTSSettings.UseCacheSystem && !bIsPreCaching) { SaveTaskAudioToCache(Task); } @@ -595,7 +608,7 @@ void UTTSManagerBase::OnGeneratedAudioChunkReceived(FTTSTask& Task, const TArray TArray UTTSManagerBase::ResampleAudio(const TArray& InputPCM) { // Check for invalid inputs or no resampling needed - if (TTSConfig->ResampleToSampleRate <= 0 || TTSConfig->AudioSampleRate <= 0 || TTSConfig->AudioNumChannels <= 0) + if (TTSConfig->GlobalTTSSettings.ResampleToSampleRate <= 0 || TTSConfig->GlobalTTSSettings.AudioSampleRate <= 0 || TTSConfig->GlobalTTSSettings.AudioNumChannels <= 0) { return InputPCM; } @@ -606,15 +619,15 @@ TArray UTTSManagerBase::ResampleAudio(const TArray& InputPCM) // Calculate step size: how much to advance in input for each output sample // Step > 1.0 means downsampling (skipping input samples) // Step < 1.0 means upsampling (interpolating between input samples) - const double Step = static_cast(TTSConfig->AudioSampleRate) / static_cast(TTSConfig->ResampleToSampleRate); + const double Step = static_cast(TTSConfig->GlobalTTSSettings.AudioSampleRate) / static_cast(TTSConfig->GlobalTTSSettings.ResampleToSampleRate); const int32 BytesPerSample = 2; // 16-bit PCM audio is 2 bytes per sample - const int32 NumSamplesInput = InputPCM.Num() / (TTSConfig->AudioNumChannels * BytesPerSample); + const int32 NumSamplesInput = InputPCM.Num() / (TTSConfig->GlobalTTSSettings.AudioNumChannels * BytesPerSample); // We want output duration roughly equal to input duration // NumSamplesOutput = NumSamplesInput / Step const int32 NumSamplesOutput = FMath::CeilToInt(NumSamplesInput / Step); - const int32 OutputSize = NumSamplesOutput * TTSConfig->AudioNumChannels * BytesPerSample; + const int32 OutputSize = NumSamplesOutput * TTSConfig->GlobalTTSSettings.AudioNumChannels * BytesPerSample; // Create output buffer TArray OutputPCM; @@ -629,7 +642,7 @@ TArray UTTSManagerBase::ResampleAudio(const TArray& InputPCM) const double Fraction = InSampleIdxF - InSampleIdx; // For each channel - for (int32 Channel = 0; Channel < TTSConfig->AudioNumChannels; ++Channel) + for (int32 Channel = 0; Channel < TTSConfig->GlobalTTSSettings.AudioNumChannels; ++Channel) { // Get the two input samples to interpolate between // Clamp indices to ensure valid range @@ -637,8 +650,8 @@ TArray UTTSManagerBase::ResampleAudio(const TArray& InputPCM) int32 InSample2 = FMath::Min(InSampleIdx + 1, NumSamplesInput - 1); // Calculate input buffer indices - int32 InIndex1 = (InSample1 * TTSConfig->AudioNumChannels + Channel) * BytesPerSample; - int32 InIndex2 = (InSample2 * TTSConfig->AudioNumChannels + Channel) * BytesPerSample; + int32 InIndex1 = (InSample1 * TTSConfig->GlobalTTSSettings.AudioNumChannels + Channel) * BytesPerSample; + int32 InIndex2 = (InSample2 * TTSConfig->GlobalTTSSettings.AudioNumChannels + Channel) * BytesPerSample; // Ensure indices are within bounds (double check) if (InIndex1 >= 0 && InIndex1 + 1 < InputPCM.Num() && InIndex2 >= 0 && InIndex2 + 1 < InputPCM.Num()) @@ -651,7 +664,7 @@ TArray UTTSManagerBase::ResampleAudio(const TArray& InputPCM) int16 OutputSample = static_cast(Sample1 + Fraction * (Sample2 - Sample1)); // Write to output buffer - int32 OutIndex = (OutSampleIdx * TTSConfig->AudioNumChannels + Channel) * BytesPerSample; + int32 OutIndex = (OutSampleIdx * TTSConfig->GlobalTTSSettings.AudioNumChannels + Channel) * BytesPerSample; if (OutIndex + 1 < OutputPCM.Num()) { OutputPCM[OutIndex] = OutputSample & 0xFF; @@ -662,7 +675,7 @@ TArray UTTSManagerBase::ResampleAudio(const TArray& InputPCM) } TTSLog(FString::Printf(TEXT("Resampled audio from %dHz to %d: %d bytes -> %d bytes"), - TTSConfig->AudioSampleRate, TTSConfig->ResampleToSampleRate, InputPCM.Num(), OutputPCM.Num())); + TTSConfig->GlobalTTSSettings.AudioSampleRate, TTSConfig->GlobalTTSSettings.ResampleToSampleRate, InputPCM.Num(), OutputPCM.Num())); return OutputPCM; } @@ -687,11 +700,11 @@ void UTTSManagerBase::RegisterAudioComponent(UAudioComponent* AudioComponent) // Create new procedural sound wave with proper initialization ProceduralSoundWave = NewObject(this); - int32 SampleRate = (TTSConfig->ResampleToSampleRate) > 0 ? TTSConfig->ResampleToSampleRate : TTSConfig->AudioSampleRate; + int32 SampleRate = (TTSConfig->GlobalTTSSettings.ResampleToSampleRate) > 0 ? TTSConfig->GlobalTTSSettings.ResampleToSampleRate : TTSConfig->GlobalTTSSettings.AudioSampleRate; ProceduralSoundWave->SetSampleRate(SampleRate); - ProceduralSoundWave->NumChannels = TTSConfig->AudioNumChannels; - ProceduralSoundWave->SetSamplesToGeneratePerCallback(SampleRate * TTSConfig->ChunkLength); - ProceduralSoundWave->SetStreamMultiplier(TTSConfig->StreamAmplitudeMultiplier); + ProceduralSoundWave->NumChannels = TTSConfig->GlobalTTSSettings.AudioNumChannels; + ProceduralSoundWave->SetSamplesToGeneratePerCallback(SampleRate * TTSConfig->GlobalTTSSettings.ChunkLength); + ProceduralSoundWave->SetStreamMultiplier(TTSConfig->GlobalTTSSettings.StreamAmplitudeMultiplier); ProceduralSoundWave->Duration = 0.0f; ProceduralSoundWave->bLooping = false; ProceduralSoundWave->bProcedural = true; @@ -704,7 +717,7 @@ void UTTSManagerBase::RegisterAudioComponent(UAudioComponent* AudioComponent) // Initialise WebRTC processing channel WebRTCChannel = MakeUnique(); - WebRTCChannel->Initialize(SampleRate, TTSConfig->AudioNumChannels); + WebRTCChannel->Initialize(SampleRate, TTSConfig->GlobalTTSSettings.AudioNumChannels); TTSLog(TEXT("WebRTC audio processing channel initialised")); } @@ -834,8 +847,8 @@ bool UTTSManagerBase::FlushToAudioQueue() { bool bQueuedAny = false; const int32 BytesPerSample = 2; // 16-bit PCM - const int32 NumChannels = TTSConfig->AudioNumChannels; - const int32 EffectiveSampleRate = (TTSConfig->ResampleToSampleRate > 0) ? TTSConfig->ResampleToSampleRate : TTSConfig->AudioSampleRate; + const int32 NumChannels = TTSConfig->GlobalTTSSettings.AudioNumChannels; + const int32 EffectiveSampleRate = (TTSConfig->GlobalTTSSettings.ResampleToSampleRate > 0) ? TTSConfig->GlobalTTSSettings.ResampleToSampleRate : TTSConfig->GlobalTTSSettings.AudioSampleRate; const int32 BytesPerFrame = BytesPerSample * NumChannels; const int32 BytesPerSecond = EffectiveSampleRate * BytesPerFrame; @@ -968,8 +981,8 @@ void UTTSManagerBase::OnAudioSample(const TArray& PCMData, int32 NumSampl } } // Broadcast with the effective sample rate, not the input rate - const int32 EffectiveSampleRate = (TTSConfig->ResampleToSampleRate > 0) ? TTSConfig->ResampleToSampleRate : TTSConfig->AudioSampleRate; - OnTTSAudioChunkForBP.Broadcast(FloatPCMData, EffectiveSampleRate, TTSConfig->AudioNumChannels); + const int32 EffectiveSampleRate = (TTSConfig->GlobalTTSSettings.ResampleToSampleRate > 0) ? TTSConfig->GlobalTTSSettings.ResampleToSampleRate : TTSConfig->GlobalTTSSettings.AudioSampleRate; + OnTTSAudioChunkForBP.Broadcast(FloatPCMData, EffectiveSampleRate, TTSConfig->GlobalTTSSettings.AudioNumChannels); } void UTTSManagerBase::AddExcludedDelimiters(const TArray& Delimiters) @@ -988,10 +1001,10 @@ void UTTSManagerBase::ProcessText(bool IsFinal) bool bIsCommaSplit = false; int32 LastSentenceEnd = 0; //This it too short, do not process so far. - if (PendingText.Len() < TTSConfig->MaxCharacterForSplit && !IsFinal) + if (PendingText.Len() < TTSConfig->GlobalTTSSettings.MaxCharacterForSplit && !IsFinal) return; - if(TTSConfig->MaxCharacterForGeneration > 0 && PendingText.Len() > TTSConfig->MaxCharacterForGeneration) + if(TTSConfig->GlobalTTSSettings.MaxCharacterForGeneration > 0 && PendingText.Len() > TTSConfig->GlobalTTSSettings.MaxCharacterForGeneration) { int32 SpaceIndex = PendingText.Find(TEXT(" "), PendingText.Len(), ESearchCase::IgnoreCase, ESearchDir::FromEnd); if(SpaceIndex > 0) @@ -1006,15 +1019,15 @@ void UTTSManagerBase::ProcessText(bool IsFinal) } } - if (TTSConfig->bCommaSplitRule != ECommaSplitRule::NeverSplit) { - for (int32 i = TTSConfig->MaxCharacterForSplit; i < Text.Len(); ++i) + if (TTSConfig->GlobalTTSSettings.bCommaSplitRule != ECommaSplitRule::NeverSplit) { + for (int32 i = TTSConfig->GlobalTTSSettings.MaxCharacterForSplit; i < Text.Len(); ++i) { TCHAR C = Text[i]; const bool IsLineBreak = (C == '\n' || C == '\r'); bool IsDelimiter = IsLineBreak || (C == '.' || C == '!' || C == '?' || C == ':'); - if (C == ',' && int32(TTSConfig->bCommaSplitRule) > int32(ECommaSplitRule::NeverSplit)) + if (C == ',' && int32(TTSConfig->GlobalTTSSettings.bCommaSplitRule) > int32(ECommaSplitRule::NeverSplit)) { - if (TTSConfig->bCommaSplitRule == ECommaSplitRule::FirstSplit && ActiveTasks.IsEmpty() && PendingTasks.IsEmpty() || int32(TTSConfig->bCommaSplitRule) >= int32(ECommaSplitRule::AlwaysSplit)) + if (TTSConfig->GlobalTTSSettings.bCommaSplitRule == ECommaSplitRule::FirstSplit && ActiveTasks.IsEmpty() && PendingTasks.IsEmpty() || int32(TTSConfig->GlobalTTSSettings.bCommaSplitRule) >= int32(ECommaSplitRule::AlwaysSplit)) { IsDelimiter = true; bIsCommaSplit = true; @@ -1071,7 +1084,7 @@ void UTTSManagerBase::ProcessText(bool IsFinal) { // Directly add to PendingTasks queue instead of redundant SentenceQueue UTTSManagerBase::AddTask(Sentence); - if (bIsCommaSplit && TTSConfig->bCommaSplitRule == ECommaSplitRule::FillWord && FMath::RandRange(1, 100 / 25) == 1 ? true : false) + if (bIsCommaSplit && TTSConfig->GlobalTTSSettings.bCommaSplitRule == ECommaSplitRule::FillWord && FMath::RandRange(1, 100 / 25) == 1 ? true : false) { UTTSManagerBase::AddTask(TEXT("..ähm..")); } @@ -1104,11 +1117,11 @@ void UTTSManagerBase::CheckGenerateAudio() int32 CurrentlyGenerating = GetCurrentlyGeneratingTaskCount_NoLock(); TTSLog(FString::Printf(TEXT("ActiveTasks: %d, CurrentlyGenerating: %d, MaxConcurrent: %d, PendingTasksEmpty: %s"), - ActiveTasks.Num(), CurrentlyGenerating, TTSConfig->MaxConcurrentGenerations, PendingTasks.IsEmpty() ? TEXT("true") : TEXT("false"))); + ActiveTasks.Num(), CurrentlyGenerating, TTSConfig->GlobalTTSSettings.MaxConcurrentGenerations, PendingTasks.IsEmpty() ? TEXT("true") : TEXT("false"))); // Launch as many tasks as possible up to MaxConcurrentGenerations // This ensures we always utilize the full concurrent capacity - while (CurrentlyGenerating < TTSConfig->MaxConcurrentGenerations && !PendingTasks.IsEmpty()) + while (CurrentlyGenerating < TTSConfig->GlobalTTSSettings.MaxConcurrentGenerations && !PendingTasks.IsEmpty()) { // If not producing, start generating audio if (CurrentState == ETTSState::Ready) @@ -1182,7 +1195,17 @@ void UTTSManagerBase::CheckPlayback() } else { - UTTSManagerBase::FlushToAudioQueue(); + if (!UTTSManagerBase::FlushToAudioQueue() && CheckAllDone()) + { + // Nothing left to play and all generation is done — finalize if audio has stopped. + // This handles the race where generation completes after the wave already underran + // (WaitingForChunks), leaving no further callback to drive the Ready transition. + const bool bIsPlaying = (RegisteredAudioComponent && RegisteredAudioComponent->IsPlaying()); + if (!bIsPlaying) + { + OnTTSManagerFinished(); + } + } } } @@ -1336,8 +1359,8 @@ static bool ParseWavPCM16(const TArray& In, int32& OutChannels, int32& Ou bool UTTSManagerBase::SaveTaskAudioToCache(const FTTSTask& Task) { if (Task.AudioData.Num() == 0 || !TTSConfig) return false; - const int32 SampleRate = (TTSConfig->ResampleToSampleRate > 0) ? TTSConfig->ResampleToSampleRate : TTSConfig->AudioSampleRate; - const int32 NumChannels = TTSConfig->AudioNumChannels; + const int32 SampleRate = (TTSConfig->GlobalTTSSettings.ResampleToSampleRate > 0) ? TTSConfig->GlobalTTSSettings.ResampleToSampleRate : TTSConfig->GlobalTTSSettings.AudioSampleRate; + const int32 NumChannels = TTSConfig->GlobalTTSSettings.AudioNumChannels; const int32 BitsPerSample = 16; TArray Wav; @@ -1374,8 +1397,8 @@ bool UTTSManagerBase::TryLoadCachedAudioIntoTask(FTTSTask& Task) return false; // Optional: validate against current config to avoid mismatched audio - const int32 ExpectedRate = (TTSConfig && TTSConfig->ResampleToSampleRate > 0) ? TTSConfig->ResampleToSampleRate : (TTSConfig ? TTSConfig->AudioSampleRate : SampleRate); - const int32 ExpectedCh = (TTSConfig ? TTSConfig->AudioNumChannels : Channels); + const int32 ExpectedRate = (TTSConfig && TTSConfig->GlobalTTSSettings.ResampleToSampleRate > 0) ? TTSConfig->GlobalTTSSettings.ResampleToSampleRate : (TTSConfig ? TTSConfig->GlobalTTSSettings.AudioSampleRate : SampleRate); + const int32 ExpectedCh = (TTSConfig ? TTSConfig->GlobalTTSSettings.AudioNumChannels : Channels); if (Channels != ExpectedCh || SampleRate != ExpectedRate) { TTSLog(FString::Printf(TEXT("Cache format mismatch (Ch=%d/%d, SR=%d/%d) - regenerating"), Channels, ExpectedCh, SampleRate, ExpectedRate)); diff --git a/Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Public/Cartesia/TTSCartesiaConfig.h b/Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Public/Cartesia/TTSCartesiaConfig.h index d9cd1cd..2545678 100644 --- a/Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Public/Cartesia/TTSCartesiaConfig.h +++ b/Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Public/Cartesia/TTSCartesiaConfig.h @@ -6,18 +6,11 @@ #include "TTSBaseConfig.h" #include "TTSCartesiaConfig.generated.h" -/** - * - */ -UCLASS() -class AVATARCORE_TTS_API UTTSCartesiaConfig : public UTTSBaseConfig +USTRUCT(BlueprintType) +struct FCartesiaTTSSettings { GENERATED_BODY() - UTTSCartesiaConfig(const FObjectInitializer& ObjectInitializer); - - public: - UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreTTS|Cartesia", meta = (ExposeOnSpawn = "true")) FString CartesiaAPIKey = ""; @@ -44,6 +37,22 @@ class AVATARCORE_TTS_API UTTSCartesiaConfig : public UTTSBaseConfig UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreTTS|Cartesia", meta = (ExposeOnSpawn = "true")) FString CartesiaContainer = TEXT("raw"); +}; + +/** + * + */ +UCLASS() +class AVATARCORE_TTS_API UTTSCartesiaConfig : public UTTSBaseConfig +{ + GENERATED_BODY() + + UTTSCartesiaConfig(const FObjectInitializer& ObjectInitializer); + + public: + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreTTS|Cartesia", meta = (ExposeOnSpawn = "true")) + FCartesiaTTSSettings CartesiaTTSSettings; FString GetHashPrefix() const override; }; diff --git a/Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Public/Elevenlabs/ElevenlabsTTSConfig.h b/Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Public/Elevenlabs/ElevenlabsTTSConfig.h index 87f2914..7ea72a3 100644 --- a/Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Public/Elevenlabs/ElevenlabsTTSConfig.h +++ b/Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Public/Elevenlabs/ElevenlabsTTSConfig.h @@ -6,18 +6,11 @@ #include "TTSBaseConfig.h" #include "ElevenlabsTTSConfig.generated.h" -/** - * - */ -UCLASS() -class AVATARCORE_TTS_API UElevenlabsTTSConfig : public UTTSBaseConfig +USTRUCT(BlueprintType) +struct FElevenlabsTTSSettings { GENERATED_BODY() - UElevenlabsTTSConfig(const FObjectInitializer& ObjectInitializer); - -public: - UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreTTS|Elevenlabs", meta = (ExposeOnSpawn = "true")) FString ElevenlabsAPIKey = ""; @@ -37,6 +30,22 @@ public: UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreTTS|Elevenlabs", meta = (ExposeOnSpawn = "true")) FString ElevenlabsURI = "api-global-preview.elevenlabs.io"; +}; + +/** + * + */ +UCLASS() +class AVATARCORE_TTS_API UElevenlabsTTSConfig : public UTTSBaseConfig +{ + GENERATED_BODY() + + UElevenlabsTTSConfig(const FObjectInitializer& ObjectInitializer); + +public: + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreTTS|Elevenlabs", meta = (ExposeOnSpawn = "true")) + FElevenlabsTTSSettings ElevenlabsTTSSettings; /* NOT RECOMMENDED ANYMORE https://help.elevenlabs.io/hc/en-us/articles/15726761640721-Can-I-reduce-API-latency // 0 (max quality) .. 4 (max latency optimization) diff --git a/Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Public/TTSBaseConfig.h b/Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Public/TTSBaseConfig.h index 72ecd26..b02803c 100644 --- a/Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Public/TTSBaseConfig.h +++ b/Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Public/TTSBaseConfig.h @@ -14,21 +14,65 @@ enum class ECommaSplitRule : uint8 { FillWord UMETA(DisplayName = "Split on every comma, but add fill words.") }; -class UTTSManagerBase; +UENUM(BlueprintType) +enum class ETTSKeepAliveRule : uint8 { + EditorOnly UMETA(DisplayName = "Only keep TTS Module open in Editor Mode"), + Never UMETA(DisplayName = "Never keep TTS Module alive."), + Always UMETA(DisplayName = "Keep TTS Module alive in Editor and Shipping mode.") +}; -/** - * - */ -UCLASS(Abstract, Blueprintable, BlueprintType) -class AVATARCORE_TTS_API UTTSBaseConfig : public UObject +UENUM(BlueprintType) +enum class ETTSLanguage : uint8 { - GENERATED_BODY() - -public: + NONE UMETA(DisplayName = "Unset"), + en UMETA(DisplayName = "English"), + fr UMETA(DisplayName = "French"), + de UMETA(DisplayName = "German"), + es UMETA(DisplayName = "Spanish"), + pt UMETA(DisplayName = "Portuguese"), + zh UMETA(DisplayName = "Chinese"), + ja UMETA(DisplayName = "Japanese"), + hi UMETA(DisplayName = "Hindi"), + it UMETA(DisplayName = "Italian"), + ko UMETA(DisplayName = "Korean"), + nl UMETA(DisplayName = "Dutch"), + pl UMETA(DisplayName = "Polish"), + ru UMETA(DisplayName = "Russian"), + sv UMETA(DisplayName = "Swedish"), + tr UMETA(DisplayName = "Turkish"), + tl UMETA(DisplayName = "Filipino"), + bg UMETA(DisplayName = "Bulgarian"), + ro UMETA(DisplayName = "Romanian"), + ar UMETA(DisplayName = "Arabic"), + cs UMETA(DisplayName = "Czech"), + el UMETA(DisplayName = "Greek"), + fi UMETA(DisplayName = "Finnish"), + hr UMETA(DisplayName = "Croatian"), + ms UMETA(DisplayName = "Malay"), + sk UMETA(DisplayName = "Slovak"), + da UMETA(DisplayName = "Danish"), + ta UMETA(DisplayName = "Tamil"), + uk UMETA(DisplayName = "Ukrainian"), + hu UMETA(DisplayName = "Hungarian"), + no UMETA(DisplayName = "Norwegian"), + vi UMETA(DisplayName = "Vietnamese"), + bn UMETA(DisplayName = "Bengali"), + th UMETA(DisplayName = "Thai"), + he UMETA(DisplayName = "Hebrew"), + ka UMETA(DisplayName = "Georgian"), + id UMETA(DisplayName = "Indonesian"), + te UMETA(DisplayName = "Telugu"), + gu UMETA(DisplayName = "Gujarati"), + kn UMETA(DisplayName = "Kannada"), + ml UMETA(DisplayName = "Malayalam"), + mr UMETA(DisplayName = "Marathi"), + pa UMETA(DisplayName = "Punjabi"), +}; - //Class of the Manager - UPROPERTY(BlueprintReadOnly, Category = "AvatarCoreTTS|Base") - TSubclassOf TSSManagerClass; +USTRUCT(BlueprintType) +struct FGlobalTTSSettings +{ + GENERATED_BODY() //If true, the audio will be cached UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreTTS|Base", meta = (ExposeOnSpawn = "true")) @@ -74,11 +118,34 @@ public: UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreTTS|Base", meta = (ExposeOnSpawn = "true")) int32 MaxCharacterForGeneration = 0; + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreTTS|Base", meta = (ExposeOnSpawn = "true")) + ETTSLanguage Language = ETTSLanguage::de; +}; + +class UTTSManagerBase; + +/** + * + */ +UCLASS(Abstract, Blueprintable, BlueprintType) +class AVATARCORE_TTS_API UTTSBaseConfig : public UObject +{ + GENERATED_BODY() + +public: + + //Class of the Manager + UPROPERTY(BlueprintReadOnly, Category = "AvatarCoreTTS|Base") + TSubclassOf TSSManagerClass; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreTTS|Settings", meta = (ExposeOnSpawn = "true")) + FGlobalTTSSettings GlobalTTSSettings; + //Helper function to generate a hash prefix so that we know if the cached audio is made with the same relevant settings) UFUNCTION() virtual FString GetHashPrefix() const { - const int32 EffectiveSR = (ResampleToSampleRate > 0) ? ResampleToSampleRate : AudioSampleRate; - return FString::Printf(TEXT("%d|%d"), AudioNumChannels, EffectiveSR); + const int32 EffectiveSR = (GlobalTTSSettings.ResampleToSampleRate > 0) ? GlobalTTSSettings.ResampleToSampleRate : GlobalTTSSettings.AudioSampleRate; + return FString::Printf(TEXT("%d|%d"), GlobalTTSSettings.AudioNumChannels, EffectiveSR); }; }; diff --git a/Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Public/TTSManagerBase.h b/Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Public/TTSManagerBase.h index 78f6777..813059e 100644 --- a/Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Public/TTSManagerBase.h +++ b/Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Public/TTSManagerBase.h @@ -107,6 +107,10 @@ public: UFUNCTION(BlueprintCallable, Category = "AvatarCoreTTS|Caching") void SetCachingEnabled(bool bEnabled); + // Set the language used for TTS generation + UFUNCTION(BlueprintCallable, Category = "AvatarCoreTTS|Settings") + void SetLanguage(ETTSLanguage NewLanguage); + // Is the TTSManager currently busy UFUNCTION(BlueprintCallable, Category = "AvatarCoreTTS|Helper") bool IsTTSBusy(); diff --git a/Unreal/Plugins/BSettings/Source/BSettings/Private/BSettingsSystem.cpp b/Unreal/Plugins/BSettings/Source/BSettings/Private/BSettingsSystem.cpp index f760f23..2524af2 100644 --- a/Unreal/Plugins/BSettings/Source/BSettings/Private/BSettingsSystem.cpp +++ b/Unreal/Plugins/BSettings/Source/BSettings/Private/BSettingsSystem.cpp @@ -5,13 +5,16 @@ #include "Misc/Paths.h" #include "HAL/FileManager.h" #include "Engine/UserDefinedEnum.h" -#include "StructUtils/UserDefinedStruct.h" +#if (ENGINE_MAJOR_VERSION == 5) && (ENGINE_MINOR_VERSION >= 6) + #include "StructUtils/UserDefinedStruct.h" +#endif #include "UObject/UnrealType.h" // FScriptArrayHelper // Global variables UBSettingsObject* UBSettingsSystem::SettingsObject = nullptr; FOnSettingsChangedDelegate UBSettingsSystem::OnSettingsChanged; FOnBSettingsErrorMulticastDelegate UBSettingsSystem::OnError; +TMap> UBSettingsSystem::EnumDisplayByType; static FString Name; static FString Directory; @@ -46,6 +49,10 @@ void UBSettingsSystem::Initialize(FSubsystemCollectionBase& Collection) Name = Settings ? Settings->FileName : TEXT("Project_Config"); Directory = FPaths::Combine(FPaths::ProjectSavedDir()); + // Load display-name map before any serialization so shipping builds, which + // lack UMETA metadata, still emit and accept the editor-baked display names. + LoadSchemaEnumMap(); + InitializeSettingsJson(); } @@ -98,7 +105,11 @@ bool UBSettingsSystem::InitializeSettingsJson() if (SettingsObject && SettingsObject->StructPtr && SettingsObject->StructPtr->PropertyLink) { const uint8* DefaultData = SettingsObject->StructPtr->GetDefaultInstance(); - FProperty *prop = SettingsObject->StructPtr->PropertyLink; + if (!DefaultData) + { + UE_LOG(LogTemp, Warning, TEXT("BSettings: GetDefaultInstance() returned null for '%s' — struct not ready yet, skipping initialization"), *SettingsObject->StructPtr->GetName()); + return false; + } InitializeValues(SettingsJson, SettingsObject->StructPtr, DefaultData); } return true; @@ -106,6 +117,11 @@ bool UBSettingsSystem::InitializeSettingsJson() void UBSettingsSystem::InitializeValues(TSharedPtr Destination, UStruct* Struct, const uint8* InStructData) { + if (!InStructData) + { + UE_LOG(LogTemp, Warning, TEXT("BSettings: InitializeValues called with null data for struct '%s' — skipping"), *Struct->GetName()); + return; + } FProperty* prop = Struct->PropertyLink; while (prop) { @@ -122,28 +138,7 @@ void UBSettingsSystem::InitializeValues(TSharedPtr Destination, USt const void* ValuePtr = RealEnumProperty->ContainerPtrToValuePtr(InStructData); const FNumericProperty* Underlying = RealEnumProperty->GetUnderlyingProperty(); const int64 RawValue = Underlying->GetSignedIntPropertyValue(ValuePtr); - - // Prefer UUserDefinedEnum DisplayNameMap -> UEnum display name -> internal name - FString OutStr; - if (const UUserDefinedEnum* UD = Cast(Enum)) - { - // Find the internal name key for this value - const FName Key = Enum->GetNameByValue(RawValue); - if (const FText* P = UD->DisplayNameMap.Find(Key)) - { - OutStr = P->ToString(); - } - } - if (OutStr.IsEmpty()) - { - OutStr = Enum->GetDisplayNameTextByValue(RawValue).ToString(); - } - if (OutStr.IsEmpty()) - { - OutStr = Enum->GetNameStringByValue(RawValue); - } - Destination->SetStringField(RealEnumProperty->GetAuthoredName(), OutStr); - + Destination->SetStringField(RealEnumProperty->GetAuthoredName(), GetEnumExportString(Enum, RawValue)); } else if (FNumericProperty* NumericProperty = CastField(prop)) { @@ -151,24 +146,7 @@ void UBSettingsSystem::InitializeValues(TSharedPtr Destination, USt { UEnum* Enum = NumericProperty->GetIntPropertyEnum(); const int64 RawValue = NumericProperty->GetSignedIntPropertyValue_InContainer(InStructData); - FString OutStr; - if (const UUserDefinedEnum* UD = Cast(Enum)) - { - const FName Key = Enum->GetNameByValue(RawValue); - if (const FText* P = UD->DisplayNameMap.Find(Key)) - { - OutStr = P->ToString(); - } - } - if (OutStr.IsEmpty()) - { - OutStr = Enum->GetDisplayNameTextByValue(RawValue).ToString(); - } - if (OutStr.IsEmpty()) - { - OutStr = Enum->GetNameStringByValue(RawValue); - } - Destination->SetStringField(NumericProperty->GetAuthoredName(), OutStr); + Destination->SetStringField(NumericProperty->GetAuthoredName(), GetEnumExportString(Enum, RawValue)); } else if (FFloatProperty* FloatProperty = CastField(prop)) { @@ -226,15 +204,7 @@ void UBSettingsSystem::InitializeValues(TSharedPtr Destination, USt UEnum* Enum = InnerEnumProp->GetEnum(); const FNumericProperty* Underlying = InnerEnumProp->GetUnderlyingProperty(); const int64 Raw = Underlying->GetSignedIntPropertyValue(ElemPtr); - FString OutStr; - if (const UUserDefinedEnum* UD = Cast(Enum)) - { - const FName Key = Enum->GetNameByValue(Raw); - if (const FText* P = UD->DisplayNameMap.Find(Key)) OutStr = P->ToString(); - } - if (OutStr.IsEmpty()) OutStr = Enum->GetDisplayNameTextByValue(Raw).ToString(); - if (OutStr.IsEmpty()) OutStr = Enum->GetNameStringByValue(Raw); - JsonArray.Add(MakeShared(OutStr)); + JsonArray.Add(MakeShared(GetEnumExportString(Enum, Raw))); } else if (FNumericProperty* InnerNum = CastField(Inner)) { @@ -242,9 +212,7 @@ void UBSettingsSystem::InitializeValues(TSharedPtr Destination, USt { UEnum* Enum = InnerNum->GetIntPropertyEnum(); const int64 Raw = InnerNum->GetSignedIntPropertyValue(ElemPtr); - FString OutStr = Enum->GetDisplayNameTextByValue(Raw).ToString(); - if (OutStr.IsEmpty()) OutStr = Enum->GetNameStringByValue(Raw); - JsonArray.Add(MakeShared(OutStr)); + JsonArray.Add(MakeShared(GetEnumExportString(Enum, Raw))); } else { @@ -329,7 +297,24 @@ void UBSettingsSystem::InitializeValues(TSharedPtr Destination, USt else { TSharedPtr NestedJson = MakeShared(); - InitializeValues(NestedJson, StructProperty->Struct, (const uint8*)DataPtr); + // For UUserDefinedStruct, use the struct's own DefaultInstance directly. + // The slot in the parent's default memory may be zeroed if the nested struct + // hadn't completed PostLoad when the parent's RecreateDefaultInstance ran + // (load-ordering issue during PreDefault phase). + const uint8* NestedData = (const uint8*)DataPtr; + if (const UUserDefinedStruct* NestedUDS = Cast(StructProperty->Struct)) + { + const uint8* UDSDefault = NestedUDS->GetDefaultInstance(); + if (UDSDefault) + { + NestedData = UDSDefault; + } + else + { + UE_LOG(LogTemp, Warning, TEXT("BSettings: nested UUserDefinedStruct '%s' has no DefaultInstance yet — values may be zeroed"), *NestedUDS->GetName()); + } + } + InitializeValues(NestedJson, StructProperty->Struct, NestedData); Destination->SetObjectField(FieldName, NestedJson); } } @@ -338,6 +323,162 @@ void UBSettingsSystem::InitializeValues(TSharedPtr Destination, USt } } +// Runtime-safe enum serializer. UMETA(DisplayName=...) is editor-only (WITH_EDITOR), +// so GetDisplayNameTextByValue returns a camelCase-split short name in shipping +// builds and no longer matches the editor-generated schema. Resolution order: +// 1. UUserDefinedEnum::DisplayNameMap (saved on the asset, always cooked) +// 2. EnumDisplayByType (schema baked in editor, loaded on Initialize) +// 3. GetDisplayNameTextByIndex (editor-only UMETA; covers the first editor run +// before a schema exists on disk) +// 4. GetNameStringByIndex (stripped member name) as the last fallback +FString UBSettingsSystem::GetEnumExportString(const UEnum* Enum, int64 Value) +{ + if (!Enum) return FString(); + const int32 Index = Enum->GetIndexByValue(Value); + if (Index == INDEX_NONE) return FString(); + + if (const UUserDefinedEnum* UD = Cast(Enum)) + { + const FName Key = Enum->GetNameByIndex(Index); + if (const FText* P = UD->DisplayNameMap.Find(Key)) + { + const FString S = P->ToString(); + if (!S.IsEmpty()) return S; + } + } + + if (const TArray* Displays = EnumDisplayByType.Find(Enum->GetName())) + { + if (Displays->IsValidIndex(Index) && !(*Displays)[Index].IsEmpty()) + { + return (*Displays)[Index]; + } + } + +#if WITH_EDITOR + const FString Display = Enum->GetDisplayNameTextByIndex(Index).ToString(); + if (!Display.IsEmpty()) return Display; +#endif + + return Enum->GetNameStringByIndex(Index); +} + +// Reads the schema file written by StructToJsonSchema and populates +// EnumDisplayByType. Looked up first in Content/Schema (the canonical location +// packaged with shipping builds) with a Saved/Schema fallback for editor convenience. +void UBSettingsSystem::LoadSchemaEnumMap() +{ + EnumDisplayByType.Empty(); + + if (Name.IsEmpty()) return; + + const FString ContentSchemaPath = FPaths::Combine(FPaths::ProjectContentDir(), TEXT("Schema"), Name + TEXT(".schema.json")); + const FString SavedSchemaPath = FPaths::Combine(Directory, TEXT("Schema"), Name + TEXT(".schema.json")); + + FString JsonString; + if (!FFileHelper::LoadFileToString(JsonString, *ContentSchemaPath) && + !FFileHelper::LoadFileToString(JsonString, *SavedSchemaPath)) + { + // No schema yet — GetEnumExportString will fall back to UMETA in editor. + return; + } + + TSharedPtr Root; + TSharedRef> Reader = TJsonReaderFactory<>::Create(JsonString); + if (!FJsonSerializer::Deserialize(Reader, Root) || !Root.IsValid()) + { + UE_LOG(LogTemp, Warning, TEXT("BSettings: failed to parse schema file for enum display lookup")); + return; + } + + const TArray>* Variables = nullptr; + if (!Root->TryGetArrayField(TEXT("Variables"), Variables)) return; + + for (const TSharedPtr& VarWrap : *Variables) + { + const TSharedPtr* WrapObj = nullptr; + if (!VarWrap.IsValid() || !VarWrap->TryGetObject(WrapObj) || !WrapObj->IsValid()) continue; + for (const auto& Pair : (*WrapObj)->Values) + { + const TSharedPtr* FieldObj = nullptr; + if (Pair.Value.IsValid() && Pair.Value->TryGetObject(FieldObj) && FieldObj->IsValid()) + { + IngestSchemaField(FieldObj->Get()); + } + } + } +} + +void UBSettingsSystem::IngestSchemaField(const FJsonObject* FieldObj) +{ + if (!FieldObj) return; + + FString Type; + FieldObj->TryGetStringField(TEXT("type"), Type); + + if (Type == TEXT("enum")) + { + FString EnumTypeName; + if (!FieldObj->TryGetStringField(TEXT("enumTypeName"), EnumTypeName) || EnumTypeName.IsEmpty()) return; + const TArray>* Values = nullptr; + if (!FieldObj->TryGetArrayField(TEXT("enum"), Values)) return; + + TArray& Dst = EnumDisplayByType.FindOrAdd(EnumTypeName); + if (!Dst.IsEmpty()) return; // first-seen wins; identical enum reused across props + Dst.Reserve(Values->Num()); + for (const TSharedPtr& V : *Values) + { + Dst.Add(V.IsValid() ? V->AsString() : FString()); + } + } + else if (Type == TEXT("struct")) + { + const TSharedPtr* Fields = nullptr; + if (!FieldObj->TryGetObjectField(TEXT("fields"), Fields) || !Fields->IsValid()) return; + for (const auto& Pair : (*Fields)->Values) + { + const TSharedPtr* Sub = nullptr; + if (Pair.Value.IsValid() && Pair.Value->TryGetObject(Sub) && Sub->IsValid()) + { + IngestSchemaField(Sub->Get()); + } + } + } + else if (Type == TEXT("array")) + { + FString InnerType; + FieldObj->TryGetStringField(TEXT("itemsType"), InnerType); + if (InnerType == TEXT("enum")) + { + FString EnumTypeName; + if (!FieldObj->TryGetStringField(TEXT("itemsEnumTypeName"), EnumTypeName) || EnumTypeName.IsEmpty()) return; + const TArray>* Values = nullptr; + if (!FieldObj->TryGetArrayField(TEXT("itemsEnum"), Values)) return; + + TArray& Dst = EnumDisplayByType.FindOrAdd(EnumTypeName); + if (!Dst.IsEmpty()) return; + Dst.Reserve(Values->Num()); + for (const TSharedPtr& V : *Values) + { + Dst.Add(V.IsValid() ? V->AsString() : FString()); + } + } + else if (InnerType == TEXT("struct")) + { + const TSharedPtr* Fields = nullptr; + if (!FieldObj->TryGetObjectField(TEXT("itemsFields"), Fields) || !Fields->IsValid()) return; + for (const auto& Pair : (*Fields)->Values) + { + const TSharedPtr* Sub = nullptr; + if (Pair.Value.IsValid() && Pair.Value->TryGetObject(Sub) && Sub->IsValid()) + { + IngestSchemaField(Sub->Get()); + } + } + } + } +} + // Helper: resolve enum value from a string using internal name, display name, and UUserDefinedEnum's DisplayNameMap bool UBSettingsSystem::ResolveEnumValueFromString(const UEnum* Enum, const FString& InString, int64& OutValue) { @@ -354,7 +495,22 @@ bool UBSettingsSystem::ResolveEnumValueFromString(const UEnum* Enum, const FStri return true; } - // Try display names (case-insensitive), skipping hidden/spacer/_MAX + // Try schema-baked display names (shipping-safe, from Content/Schema/*.schema.json) + if (const TArray* Displays = EnumDisplayByType.Find(Enum->GetName())) + { + for (int32 i = 0; i < Displays->Num(); ++i) + { + if (S.Equals((*Displays)[i], ESearchCase::IgnoreCase)) + { + OutValue = Enum->GetValueByIndex(i); + return true; + } + } + } + + // Try display names (case-insensitive), skipping hidden/spacer/_MAX. + // GetDisplayNameTextByIndex consults UMETA metadata in editor builds; in + // shipping it falls back to a camelCase-split short name. const int32 Count = Enum->NumEnums(); for (int32 i = 0; i < Count; ++i) { @@ -433,13 +589,7 @@ bool UBSettingsSystem::LoadJsonFromFile() int64 Raw; if (ResolveEnumValueFromString(PropEnum, ValString, Raw)) { - // Canonicalize to display name for readability (fallback to internal if empty) - FString Canonical = PropEnum->GetDisplayNameTextByValue(Raw).ToString(); - if (Canonical.IsEmpty()) - { - Canonical = PropEnum->GetNameStringByValue(Raw); - } - SettingsJson->SetStringField(It.Key(), Canonical); + SettingsJson->SetStringField(It.Key(), GetEnumExportString(PropEnum, Raw)); } else { @@ -583,6 +733,9 @@ bool UBSettingsSystem::SaveJsonToFile() if (bSaved && SettingsObject && SettingsObject->StructPtr) { StructToJsonSchema(SettingsObject->StructPtr, Name); + // Refresh the in-memory display map so subsequent exports in the same + // editor session pick up any enum display-name edits that just landed. + LoadSchemaEnumMap(); } return bSaved; @@ -1115,6 +1268,16 @@ TSharedPtr UBSettingsSystem::BuildFieldSchema(const FProperty* Prop JsonEnumVals.Add(MakeShared(Val)); } FieldObj->SetArrayField(TEXT("enum"), JsonEnumVals); + + // Record the UEnum name so shipping builds can rebuild the display map + // via LoadSchemaEnumMap (UMETA metadata isn't available without WITH_EDITOR). + const UEnum* EnumRef = nullptr; + if (const FEnumProperty* EnumProp = CastField(Prop)) EnumRef = EnumProp->GetEnum(); + else if (const FByteProperty* ByteProp = CastField(Prop)) EnumRef = ByteProp->Enum; + if (EnumRef) + { + FieldObj->SetStringField(TEXT("enumTypeName"), EnumRef->GetName()); + } } // Struct: recurse into sub-fields @@ -1152,6 +1315,14 @@ TSharedPtr UBSettingsSystem::BuildFieldSchema(const FProperty* Prop JsonEnumVals.Add(MakeShared(Val)); } FieldObj->SetArrayField(TEXT("itemsEnum"), JsonEnumVals); + + const UEnum* EnumRef = nullptr; + if (const FEnumProperty* EnumProp = CastField(Inner)) EnumRef = EnumProp->GetEnum(); + else if (const FByteProperty* ByteProp = CastField(Inner)) EnumRef = ByteProp->Enum; + if (EnumRef) + { + FieldObj->SetStringField(TEXT("itemsEnumTypeName"), EnumRef->GetName()); + } } else if (InnerType == TEXT("struct")) { @@ -1320,7 +1491,7 @@ bool UBSettingsSystem::StructToJsonSchema(const UStruct* StructType, const FStri // Save the schema JSON to file (UTF-8 without BOM) if (!FFileHelper::SaveStringToFile(Output, *FilePath, FFileHelper::EEncodingOptions::ForceUTF8WithoutBOM)) { - UE_LOG(LogTemp, Error, TEXT("ExportStructSchema: Failed to write file: %s"), *FilePath); + UE_LOG(LogTemp, Error, TEXT("Failed to write file: %s"), *FilePath); return false; } diff --git a/Unreal/Plugins/BSettings/Source/BSettings/Public/BSettingsSystem.h b/Unreal/Plugins/BSettings/Source/BSettings/Public/BSettingsSystem.h index 4410c9b..ad4a9b1 100644 --- a/Unreal/Plugins/BSettings/Source/BSettings/Public/BSettingsSystem.h +++ b/Unreal/Plugins/BSettings/Source/BSettings/Public/BSettingsSystem.h @@ -68,6 +68,13 @@ class BSETTINGS_API UBSettingsSystem : public UGameInstanceSubsystem static UBSettingsObject* SettingsObject; static TSharedPtr SettingsJson; + // Editor-baked display-name map loaded from Content/Schema/{Name}.schema.json. + // Keyed by UEnum object name (e.g. "ESTTTranscriptionType"); the value holds the + // enum entries in the same index order as UEnum (skipping _MAX). This is the + // shipping-safe source of UMETA(DisplayName=...) strings since UMETA metadata is + // compiled out of cooked builds. + static TMap> EnumDisplayByType; + protected: UFUNCTION(BlueprintCallable, Category = "BSettings") static UEnum* GetEnumByPropertyName(FString PropName); @@ -478,15 +485,7 @@ public: if (EnumProperty->GetEnum() == SettingsEnum) { const int64 Raw = EnumProperty->GetUnderlyingProperty()->GetSignedIntPropertyValue(ValPtr); - FString OutStr; - if (const UUserDefinedEnum* UD = Cast(SettingsEnum)) - { - const FName Key = SettingsEnum->GetNameByValue(Raw); - if (const FText* P = UD->DisplayNameMap.Find(Key)) OutStr = P->ToString(); - } - if (OutStr.IsEmpty()) OutStr = SettingsEnum->GetDisplayNameTextByValue(Raw).ToString(); - if (OutStr.IsEmpty()) OutStr = SettingsEnum->GetNameStringByValue(Raw); - SettingsJson->SetStringField(Name, OutStr); + SettingsJson->SetStringField(Name, GetEnumExportString(SettingsEnum, Raw)); } else { @@ -500,15 +499,7 @@ public: const int64 Raw = NumericProperty->GetSignedIntPropertyValue(ValPtr); if (SettingsEnum->IsValidEnumValue(Raw)) { - FString OutStr; - if (const UUserDefinedEnum* UD = Cast(SettingsEnum)) - { - const FName Key = SettingsEnum->GetNameByValue(Raw); - if (const FText* P = UD->DisplayNameMap.Find(Key)) OutStr = P->ToString(); - } - if (OutStr.IsEmpty()) OutStr = SettingsEnum->GetDisplayNameTextByValue(Raw).ToString(); - if (OutStr.IsEmpty()) OutStr = SettingsEnum->GetNameStringByValue(Raw); - SettingsJson->SetStringField(Name, OutStr); + SettingsJson->SetStringField(Name, GetEnumExportString(SettingsEnum, Raw)); } else { @@ -613,15 +604,7 @@ public: { void* ElemPtr = Helper.GetRawPtr(i); const int64 Raw = InnerEnum->GetUnderlyingProperty()->GetSignedIntPropertyValue(ElemPtr); - FString OutStr; - if (const UUserDefinedEnum* UD = Cast(Enum)) - { - const FName Key = Enum->GetNameByValue(Raw); - if (const FText* P = UD->DisplayNameMap.Find(Key)) OutStr = P->ToString(); - } - if (OutStr.IsEmpty()) OutStr = Enum->GetDisplayNameTextByValue(Raw).ToString(); - if (OutStr.IsEmpty()) OutStr = Enum->GetNameStringByValue(Raw); - Out.Add(MakeShared(OutStr)); + Out.Add(MakeShared(GetEnumExportString(Enum, Raw))); } SettingsJson->SetArrayField(Name, Out); *(bool*)RESULT_PARAM = true; @@ -639,9 +622,7 @@ public: { void* ElemPtr = Helper.GetRawPtr(i); const int64 Raw = InnerNum->GetSignedIntPropertyValue(ElemPtr); - FString OutStr = Enum->GetDisplayNameTextByValue(Raw).ToString(); - if (OutStr.IsEmpty()) OutStr = Enum->GetNameStringByValue(Raw); - Out.Add(MakeShared(OutStr)); + Out.Add(MakeShared(GetEnumExportString(Enum, Raw))); } SettingsJson->SetArrayField(Name, Out); *(bool*)RESULT_PARAM = true; @@ -760,6 +741,17 @@ public: private: static bool ResolveEnumValueFromString(const UEnum* Enum, const FString& InString, int64& OutValue); + // Runtime-safe enum-to-string serializer. UMETA(DisplayName=...) is editor-only, + // so at shipping runtime it decays to a camelCase-split short name and no longer + // matches the editor-generated schema. This helper prefers cooked storage: + // UUserDefinedEnum::DisplayNameMap for Blueprint enums, the schema map baked at + // editor cook time for C++ UENUMs, then UMETA (editor only), then stripped name. + static FString GetEnumExportString(const UEnum* Enum, int64 Value); + // Populates EnumDisplayByType by parsing Content/Schema/{Name}.schema.json (with + // Saved/Schema as fallback). Called at Initialize and after SaveJsonToFile + // regenerates the schema in editor. + static void LoadSchemaEnumMap(); + static void IngestSchemaField(const FJsonObject* FieldObj); static FString GetSchemaType(const FProperty* Prop); static void CollectEnumValues(const FProperty* Prop, TArray& OutValues); static TSharedPtr BuildFieldSchema(const FProperty* Prop); diff --git a/Unreal/Plugins/BTools/Source/BTools/Private/BToolsBPLibrary.cpp b/Unreal/Plugins/BTools/Source/BTools/Private/BToolsBPLibrary.cpp index b8d71d2..bd81e9e 100644 --- a/Unreal/Plugins/BTools/Source/BTools/Private/BToolsBPLibrary.cpp +++ b/Unreal/Plugins/BTools/Source/BTools/Private/BToolsBPLibrary.cpp @@ -171,6 +171,66 @@ int UBToolsBPLibrary::KillProcessesByPath(FString Dir) #endif } +bool UBToolsBPLibrary::KillProcessesByID(int ProcessID) +{ +#if PLATFORM_WINDOWS + if (ProcessID <= 0) + { + return false; + } + + // Snapshot all processes once and build the full kill list (target + all descendants) + const HANDLE Snapshot = ::CreateToolhelp32Snapshot(TH32CS_SNAPPROCESS, 0); + if (Snapshot == INVALID_HANDLE_VALUE) + { + return false; + } + + TArray ToKill; + ToKill.Add(static_cast(ProcessID)); + + // Single pass: keep expanding until no new children are found + bool bExpanded = true; + while (bExpanded) + { + bExpanded = false; + PROCESSENTRY32 Entry = {}; + Entry.dwSize = sizeof(PROCESSENTRY32); + if (::Process32First(Snapshot, &Entry)) + { + do + { + if (ToKill.Contains(Entry.th32ParentProcessID) && !ToKill.Contains(Entry.th32ProcessID)) + { + ToKill.Add(Entry.th32ProcessID); + bExpanded = true; + } + } while (::Process32Next(Snapshot, &Entry)); + } + } + ::CloseHandle(Snapshot); + + // Kill children first (reverse order so leaves die before parents) + bool bAnyKilled = false; + for (int32 i = ToKill.Num() - 1; i >= 0; --i) + { + const HANDLE Handle = ::OpenProcess(PROCESS_TERMINATE, false, ToKill[i]); + if (Handle) + { + if (::TerminateProcess(Handle, 0)) + { + bAnyKilled = true; + } + ::CloseHandle(Handle); + } + } + + return bAnyKilled; +#else + return false; +#endif +} + bool UBToolsBPLibrary::IsProgramRunning(FTheProcHandle procHandle) { return FPlatformProcess::IsApplicationRunning(procHandle.processID); @@ -263,6 +323,33 @@ FString UBToolsBPLibrary::GetDesktopPath() #endif } +FString UBToolsBPLibrary::ConvertToEasyLanguage(const FString& OriginalString) +{ + FString Result; + Result.Reserve(OriginalString.Len() + 32); // small buffer to avoid reallocations + + for (int32 i = 0; i < OriginalString.Len(); i++) + { + TCHAR CurrentChar = OriginalString[i]; + Result.AppendChar(CurrentChar); + + // Sentence-ending punctuation + if (CurrentChar == '.' || CurrentChar == '!' || CurrentChar == '?' || CurrentChar == '-' || CurrentChar == ':' || CurrentChar == ';') + { + // Avoid multiple line breaks + if (i + 1 < OriginalString.Len() && OriginalString[i + 1] != '\n') + { + Result.AppendChar('\n'); + } + } + } + + // Clean up: remove duplicate line breaks + Result.ReplaceInline(TEXT("\n\n"), TEXT("\n")); + + return Result; +} + bool UBToolsBPLibrary::ContainsOneOf(FString SearchIn, TArray Substrings, bool UseCase, bool SearchFromEnd, FString& FoundString) { for (auto Substring : Substrings) @@ -590,6 +677,44 @@ void UBToolsBPLibrary::CheckForSteeringSignals(FString Source, TArray& ParsedText = Result; } +bool UBToolsBPLibrary::ExtractTagValue(const FString& Source, const FString& Tag, FString& Value) +{ + Value.Empty(); + + if (Source.IsEmpty() || Tag.IsEmpty()) + { + return false; + } + + const FString OpenTag = FString::Printf(TEXT("<%s>"), *Tag); + const FString CloseTag = FString::Printf(TEXT(""), *Tag); + + const int32 OpenTagIndex = Source.Find(OpenTag); + + if (OpenTagIndex == INDEX_NONE) + { + return false; + } + + const int32 ContentStartIndex = OpenTagIndex + OpenTag.Len(); + + const int32 CloseTagIndex = Source.Find( + CloseTag, + ESearchCase::IgnoreCase, + ESearchDir::FromStart, + ContentStartIndex + ); + + if (CloseTagIndex == INDEX_NONE) + { + return false; + } + + Value = Source.Mid(ContentStartIndex, CloseTagIndex - ContentStartIndex); + + return true; +} + bool UBToolsBPLibrary::IsSentenceTerminator(char charToCheck) { return (charToCheck == '.' || charToCheck == '!' || charToCheck == '?'); diff --git a/Unreal/Plugins/BTools/Source/BTools/Public/BToolsBPLibrary.h b/Unreal/Plugins/BTools/Source/BTools/Public/BToolsBPLibrary.h index eb629d6..041414d 100644 --- a/Unreal/Plugins/BTools/Source/BTools/Public/BToolsBPLibrary.h +++ b/Unreal/Plugins/BTools/Source/BTools/Public/BToolsBPLibrary.h @@ -143,6 +143,9 @@ public: UFUNCTION(BlueprintCallable, Category = "BTools") static int KillProcessesByPath(FString Dir); + UFUNCTION(BlueprintCallable, Category = "BTools") + static bool KillProcessesByID(int ProcessID); + UFUNCTION(BlueprintCallable, Category = "BTools") static bool IsProgramRunning(FTheProcHandle procHandle); @@ -158,6 +161,14 @@ public: // String Functions + /** + * Format String to easy language + * @param OriginalString + * @return ConvertedString + */ + UFUNCTION(BlueprintCallable, Category = "BTools") + static FString ConvertToEasyLanguage(const FString& OriginalString); + /** * Tests if the search string contains any of the substrings * @param SearchIn The string to search in @@ -274,6 +285,21 @@ public: UFUNCTION(BlueprintCallable, Category = "BTools") static void CheckForSteeringSignals(FString Source, TArray& SteeringSignals, FString& ParsedText, bool IncludeStandardBracket = true); + /** + * Extracts the value inside a specified XML-style tag. + * Example: + * Source: "Find reservation" + * Tag: "title" + * Result: "Find reservation" + * + * @param Source The source string containing the tag + * @param Tag The tag name to search for + * @param Value The extracted value inside the tag + * @return True if the tag was found and parsed successfully + */ + UFUNCTION(BlueprintCallable, Category = "BTools") + static bool ExtractTagValue(const FString & Source, const FString & Tag, FString & Value); + /** * Check if the char provided is an end of a sentence (. ! ?) */ diff --git a/Unreal/SyncAvatarCore.bat b/Unreal/SyncAvatarCore.bat index 520e24b..9704a9a 100644 --- a/Unreal/SyncAvatarCore.bat +++ b/Unreal/SyncAvatarCore.bat @@ -46,9 +46,7 @@ if not exist "%SRC_ROOT%\" ( exit /b 2 ) -rem Timestamped log file -for /f %%I in ('powershell -NoProfile -Command "Get-Date -Format yyyyMMdd_HHmmss"') do set "TS=%%I" -set "LOG_FILE=%DST_ROOT%\robocopy_sync_%TS%.log" + rem List of folders to mirror set "FOLDERS=Plugins\AvatarCore_AI Plugins\AvatarCore_Manager Plugins\AvatarCore_MetaHuman Plugins\AvatarCore_STT Plugins\AvatarCore_TTS Plugins\BLogger Plugins\BSettings Plugins\BTools Plugins\RuntimeMetaHumanLipSync_5.6 Content\Project" @@ -67,45 +65,36 @@ for %%F in (%FOLDERS%) do ( if not exist "!SRC!\" ( echo WARNING: Source folder missing, skipping: !SRC! - echo WARNING: Source folder missing, skipping: !SRC!>>"%LOG_FILE%" - echo.>>"%LOG_FILE%" - pause - goto :continue - ) + ) else ( + if not exist "!DST!\" ( + mkdir "!DST!" 2>nul + ) - if not exist "!DST!\" ( - mkdir "!DST!" 2>nul - ) + robocopy "!SRC!" "!DST!" /MIR /DCOPY:DAT /COPY:DAT /R:2 /W:1 /FFT /Z /NP /FP /TS /TEE + set "RC=!ERRORLEVEL!" - robocopy "!SRC!" "!DST!" /MIR /DCOPY:DAT /COPY:DAT /R:2 /W:1 /FFT /Z /NP /FP /TS /TEE /LOG+:"%LOG_FILE%" - set "RC=!ERRORLEVEL!" + echo Robocopy exit code: !RC! - if !RC! GEQ 8 ( - echo ERROR: Robocopy reported failure for %%F (exit code !RC!) - set "ANY_FAIL=1" + if !RC! GEQ 8 ( + echo ERROR: Robocopy reported failure for %%F ^(exit code !RC!^) + set "ANY_FAIL=1" + ) ) - echo.>>"%LOG_FILE%" + echo. +) - :continue +if "!ANY_FAIL!"=="1" ( + echo ============================================================ + echo Sync failed! + echo ============================================================ + pause + exit /b 8 ) echo ============================================================ echo Sync finished. -echo Review log for removed files marked as: -echo *EXTRA File -echo *EXTRA Dir -echo. -echo Log file: -echo %LOG_FILE% echo ============================================================ -if "%ANY_FAIL%"=="1" ( - echo Log file: - echo %LOG_FILE% - echo. - pause - exit /b 8 -) pause -exit /b 0 +exit /b 0 \ No newline at end of file