diff --git a/Unreal/Config/DefaultInput.ini b/Unreal/Config/DefaultInput.ini index 322e5cd..6de27d5 100644 --- a/Unreal/Config/DefaultInput.ini +++ b/Unreal/Config/DefaultInput.ini @@ -110,6 +110,7 @@ DoubleClickTime=0.200000 +ActionMappings=(ActionName="State7",bShift=False,bCtrl=False,bAlt=False,bCmd=False,Key=Seven) +ActionMappings=(ActionName="State8",bShift=False,bCtrl=False,bAlt=False,bCmd=False,Key=Eight) +ActionMappings=(ActionName="State9",bShift=False,bCtrl=False,bAlt=False,bCmd=False,Key=Nine) ++ActionMappings=(ActionName="Button",bShift=False,bCtrl=False,bAlt=False,bCmd=False,Key=GenericUSBController_Button12) DefaultPlayerInputClass=/Script/EnhancedInput.EnhancedPlayerInput DefaultInputComponentClass=/Script/EnhancedInput.EnhancedInputComponent DefaultTouchInterface=/Engine/MobileResources/HUD/DefaultVirtualJoysticks.DefaultVirtualJoysticks diff --git a/Unreal/Content/Project/BP/BP_Project_Manager.uasset b/Unreal/Content/Project/BP/BP_Project_Manager.uasset index 6841412..06702a5 100644 --- a/Unreal/Content/Project/BP/BP_Project_Manager.uasset +++ b/Unreal/Content/Project/BP/BP_Project_Manager.uasset @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:dcd421a0daa75f4b54ae15bf923bf2cc36a80631783aa65d20b68c126aa72706 -size 2581241 +oid sha256:20342cf072703b4803cdff69c0da855a6b9aaaa07108b1af5535ffcfeeb71e75 +size 2626029 diff --git a/Unreal/Content/Project/BP/EnumsAndStructs/S_DEMO_Settings.uasset b/Unreal/Content/Project/BP/EnumsAndStructs/S_DEMO_Settings.uasset index 115cec7..a4b0704 100644 --- a/Unreal/Content/Project/BP/EnumsAndStructs/S_DEMO_Settings.uasset +++ b/Unreal/Content/Project/BP/EnumsAndStructs/S_DEMO_Settings.uasset @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:356080766cdceed9247b7471b959d5a582f349f2d821d26fdae06b5e54806dd7 -size 46007 +oid sha256:5a9442b8fe26bfdee20400d9a68bc2c80b01132cf917a3d81f4cb1f0bcf6409d +size 52416 diff --git a/Unreal/Content/Project/Widgets/ChildWidgets/W_ToolcallEntry.uasset b/Unreal/Content/Project/Widgets/ChildWidgets/W_ToolcallEntry.uasset new file mode 100644 index 0000000..e1ddeae --- /dev/null +++ b/Unreal/Content/Project/Widgets/ChildWidgets/W_ToolcallEntry.uasset @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3a19e8a393a2e163d5e9c92477e2d5f1ff402f035409833f5799f6db732b3052 +size 56988 diff --git a/Unreal/Content/Project/Widgets/Debug/W_DebugStateWidget.uasset b/Unreal/Content/Project/Widgets/Debug/W_DebugStateWidget.uasset new file mode 100644 index 0000000..77a7fed --- /dev/null +++ b/Unreal/Content/Project/Widgets/Debug/W_DebugStateWidget.uasset @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e82bb9ffa256eab6d0a17a5e1cd702b7cb91ffa87600fc0cd4bdba8b85551015 +size 53421 diff --git a/Unreal/Content/Project/Widgets/W_DialogueBox.uasset b/Unreal/Content/Project/Widgets/W_DialogueBox.uasset index defb63e..a087a45 100644 --- a/Unreal/Content/Project/Widgets/W_DialogueBox.uasset +++ b/Unreal/Content/Project/Widgets/W_DialogueBox.uasset @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:5c4b4e90d49269523f2b5974956ff685be11976c114325748ecf77759989750d -size 990921 +oid sha256:ee923ac4fff8c1f53bb32151fef8dde3ed3ce24da1e4cd3fb469a03a741e5e3f +size 971640 diff --git a/Unreal/Content/Project/Widgets/W_Toolcall.uasset b/Unreal/Content/Project/Widgets/W_Toolcall.uasset new file mode 100644 index 0000000..2760c1a --- /dev/null +++ b/Unreal/Content/Project/Widgets/W_Toolcall.uasset @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:79e819f1e541ae2a89a10abd89bdd999c733e412c53f93c5c32047bfb5dfd9c0 +size 180493 diff --git a/Unreal/Content/Project/Widgets/W_Toolcall_Simple.uasset b/Unreal/Content/Project/Widgets/W_Toolcall_Simple.uasset new file mode 100644 index 0000000..27334b7 --- /dev/null +++ b/Unreal/Content/Project/Widgets/W_Toolcall_Simple.uasset @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f774df7810438a84024db46fabf269155cdcaa8007c4f26e0c9f766cf7e885ed +size 112002 diff --git a/Unreal/Content/SPIE/BP/BP_SPIE_Manager_Child.uasset b/Unreal/Content/SPIE/BP/BP_SPIE_Manager_Child.uasset index 0bc1cad..6b48524 100644 --- a/Unreal/Content/SPIE/BP/BP_SPIE_Manager_Child.uasset +++ b/Unreal/Content/SPIE/BP/BP_SPIE_Manager_Child.uasset @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:adb2d8a6844f40ee41e217731f42fa64128a85d3eb5bc0d2327436619750cfbd -size 524584 +oid sha256:f24210c1b855d4123b721d33833a3f1fdfd69bc4e53048383337c4fa8c249026 +size 455181 diff --git a/Unreal/Content/SPIE/BP/S_SPIE_ConfigSettings.uasset b/Unreal/Content/SPIE/BP/S_SPIE_ConfigSettings.uasset index 327d15d..9abf561 100644 --- a/Unreal/Content/SPIE/BP/S_SPIE_ConfigSettings.uasset +++ b/Unreal/Content/SPIE/BP/S_SPIE_ConfigSettings.uasset @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:18fe83b2a4c0d27ddd072b5d16c79e3541aa61444cb74e728db06fadb422f008 -size 69998 +oid sha256:7e482fbbea69e92d53fa65f9ed7ed1d37d179efc8ba303f614ec414423e2fd76 +size 73565 diff --git a/Unreal/Content/Schema/Spie_Config.schema.json b/Unreal/Content/Schema/Spie_Config.schema.json index ff21900..c19af2a 100644 --- a/Unreal/Content/Schema/Spie_Config.schema.json +++ b/Unreal/Content/Schema/Spie_Config.schema.json @@ -291,21 +291,123 @@ } }, { - "AzureSpeechService_API": + "STTAzureSettings": { - "type": "string", - "tooltip": "Encrypted Azure API Key", - "default": "0Gc0wvOF2tTCr4APvZkDaieKCGeBR9c5EbC5utXgVWZUqu4IAEf6NQ721iHNu74SwfDoYSBGJHLmCbcXVDP+F4HKnwsfHr7WYi9Gv+CjJ7/UrOygkqlrP05hbHBrJiPLWDv4Gw==", - "category": "STT Settings" + "type": "struct", + "tooltip": "Settigns to configure the Mircrosoft Azure STT module", + "fields": + { + "AzureAPIKey": + { + "type": "string" + }, + "AzureRegion": + { + "type": "string" + } + }, + "default": + { + "AzureAPIKey": "0Gc0wvOF2tTCr4APvZkDaieKCGeBR9c5EbC5utXgVWZUqu4IAEf6NQ721iHNu74SwfDoYSBGJHLmCbcXVDP+F4HKnwsfHr7WYi9Gv+CjJ7/UrOygkqlrP05hbHBrJiPLWDv4Gw==", + "AzureRegion": "germanywestcentral" + }, + "category": "STT" } }, { - "AzureSpeechService_Region": + "STTWhisperSettings": { - "type": "string", - "tooltip": "Azure Region", - "default": "germanywestcentral", - "category": "STT Settings" + "type": "struct", + "tooltip": "Settigns to configure the OpenAI Whisper STT module", + "fields": + { + "OpenAI_API_Key": + { + "type": "string" + }, + "WhisperURL": + { + "type": "string" + }, + "Model": + { + "type": "enum", + "enum": [ + "Whisper-1", + "4o Transcribe Mini", + "4o Transcribe" + ], + "enumTypeName": "EOpenAITranscriptionModel" + }, + "MinDuration": + { + "type": "float" + } + }, + "default": + { + "OpenAI_API_Key": "UjzfgavJ45lCu+oB2vVAsKNbPT+k3XCv7t69Og6j0LmwxhD3OK5WDBxUvgKnuDrz3xuNHg==", + "WhisperURL": "api.openai.com/v1/audio/transcriptions", + "Model": "4o Transcribe", + "MinDuration": 0.75 + }, + "category": "STT" + } + }, + { + "STTParakeetSettings": + { + "type": "struct", + "tooltip": "Settigns to configure the nVidia parakeet STT module", + "fields": + { + "PythonPath": + { + "type": "string" + }, + "PretrainedModel": + { + "type": "string" + }, + "Host": + { + "type": "string" + }, + "Port": + { + "type": "integer" + }, + "KeepAliveRule": + { + "type": "enum", + "enum": [ + "Only keep Module open in Editor Mode", + "Never keep Module alive.", + "Keep Module alive in Editor and Shipping mode." + ], + "enumTypeName": "EKeepAliveRule" + }, + "Device": + { + "type": "string" + }, + "UpdateIntervalSec": + { + "type": "float", + "tooltip": "How often (seconds) the Python server produces intermediate transcription updates" + } + }, + "default": + { + "PythonPath": "python", + "PretrainedModel": "nvidia/parakeet-tdt-0.6b-v3", + "Host": "127.0.0.1", + "Port": 40200, + "KeepAliveRule": "Only keep Module open in Editor Mode", + "Device": "cuda:0", + "UpdateIntervalSec": 0.5 + }, + "category": "STT" } }, { @@ -562,7 +664,7 @@ "Marathi", "Punjabi" ], - "itemsEnumTypeName": "ESTTLanguage" + "itemsEnumTypeName": "ELanguage" }, "STTReplacements": { @@ -785,11 +887,6 @@ "type": "struct", "fields": { - "bUseModeration": - { - "type": "boolean", - "tooltip": "Check user transcription for inappropriate behaviour first (adds a delay!)" - }, "bUseMCPServer": { "type": "boolean", @@ -826,7 +923,6 @@ }, "default": { - "bUseModeration": false, "bUseMCPServer": false, "AIModelAudioOutput": true, "MaxTokens": 1500, @@ -991,7 +1087,7 @@ "Marathi", "Punjabi" ], - "enumTypeName": "ETTSLanguage" + "enumTypeName": "ELanguage" } }, "default": diff --git a/Unreal/Plugins/AvatarCore_AI/AvatarCore_AI.uplugin b/Unreal/Plugins/AvatarCore_AI/AvatarCore_AI.uplugin index a6db854..fbb0aed 100644 --- a/Unreal/Plugins/AvatarCore_AI/AvatarCore_AI.uplugin +++ b/Unreal/Plugins/AvatarCore_AI/AvatarCore_AI.uplugin @@ -5,7 +5,7 @@ "FriendlyName": "AvatarCore AI", "Description": "A wrapper for OpenAI Assistents", "Category": "Other", - "CreatedBy": "b.RexGmbh", + "CreatedBy": "b.ReX Gmbh", "CreatedByURL": "", "DocsURL": "", "MarketplaceURL": "", @@ -29,6 +29,10 @@ { "Name": "BTools", "Enabled": true + }, + { + "Name": "AvatarCore_Shared", + "Enabled": true } ] } \ No newline at end of file diff --git a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/AvatarCore_AI.Build.cs b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/AvatarCore_AI.Build.cs index 8968b14..09f80e2 100644 --- a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/AvatarCore_AI.Build.cs +++ b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/AvatarCore_AI.Build.cs @@ -9,6 +9,11 @@ public class AvatarCore_AI : ModuleRules { PCHUsage = ModuleRules.PCHUsageMode.UseExplicitOrSharedPCHs; + if (Target.bBuildEditor) + { + PublicDependencyModuleNames.Add("UnrealEd"); + } + PublicIncludePaths.AddRange( new string[] { // ... add public include paths required here ... @@ -51,7 +56,8 @@ public class AvatarCore_AI : ModuleRules "SSL", "Json", "Projects", // Required for IPluginManager - "BTools" + "BTools", + "AvatarCore_Shared", // ... add private dependencies that you statically link with here ... } ); diff --git a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/AIBaseManager.cpp b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/AIBaseManager.cpp index 2a6a52d..9e3bf19 100644 --- a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/AIBaseManager.cpp +++ b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/AIBaseManager.cpp @@ -1,6 +1,7 @@ // Fill out your copyright notice in the Description page of Project Settings. #include "AIBaseManager.h" +#include "FL_AvatarCoreShared.h" #include "Async/Async.h" void UAIBaseManager::InitAIManager(UAIBaseConfig* AIConfig, bool DebugMode, AActor* InWorldReferenceActor) @@ -77,6 +78,11 @@ void UAIBaseManager::InitAIManagerChild(UAIBaseConfig* AIConfig, AActor* InWorld SetNewState(EAvatarCoreAIState::Ready); } +void UAIBaseManager::SetAILanguage(ELanguage NewLanguage) +{ + ForcedLanguage = NewLanguage; +} + void UAIBaseManager::DeinitAIManager() { PreviousMessages.Empty(); @@ -166,7 +172,13 @@ void UAIBaseManager::SendResponse(FAIMessage Message, bool NotifyDelay) BroadcastAILog(FString::Printf(TEXT("AI Manager sent question/response: %s"), *Message.Message)); if (NotifyDelay) UAIBaseManager::StartDelayedAnswerTimer(); + + //Add a fixed language tag. But only to the current message not to the stack. Do we really need this? Or does the system prompt is already enough? + FAIMessage currentMessage = Message; + if (ForcedLanguage != ELanguage::NONE && currentMessage.Role == EAvatarCoreAIPromptRole::User) + currentMessage.Message = "[Answer in " + UFL_AvatarCoreShared::LanguageToString(ForcedLanguage) + "]" + currentMessage.Message; SendResponseChild(Message, NotifyDelay); + if (Message.Role == EAvatarCoreAIPromptRole::User) AddMessageToArray(Message); else if (Message.Role == EAvatarCoreAIPromptRole::Tool) @@ -186,10 +198,17 @@ void UAIBaseManager::RepeatText(FString TextToRepeat, bool DoRephrase) AnswerCache.Empty(); ResponseID++; FString Instruction; - if (DoRephrase) - Instruction = "[REPHRASE] " + TextToRepeat; + if (DoRephrase) { + if(ForcedLanguage == ELanguage::NONE) + Instruction = "[REPHRASE] " + TextToRepeat; + else + Instruction = "[REPHRASE in " + UFL_AvatarCoreShared::LanguageToString(ForcedLanguage) + "] " + TextToRepeat; + } else - Instruction = "[REPEAT] " + TextToRepeat; + if (ForcedLanguage == ELanguage::NONE) + Instruction = "[REPEAT] " + TextToRepeat; + else + Instruction = "[REPEAT in " + UFL_AvatarCoreShared::LanguageToString(ForcedLanguage) + "] " + TextToRepeat; FAIMessage tmpPrompt; tmpPrompt.Message = Instruction; tmpPrompt.Role = EAvatarCoreAIPromptRole::System; @@ -317,7 +336,6 @@ void UAIBaseManager::RunMCPCommand(FString CommandName, FString Payload, FString BroadcastAILog(FString::Printf(TEXT("Running Command '%s' with payload %s"), *CommandName, *Payload), true); SetNewState(EAvatarCoreAIState::GettingInfo); - functionCallRunning = true; if (FoundClass) { @@ -329,7 +347,7 @@ void UAIBaseManager::RunMCPCommand(FString CommandName, FString Payload, FString UMCPUnrealCommand* Cmd = NewObject(this, FoundClass); Cmd->Id = ToolCallId; Cmd->SetWorldContext(WorldReferenceActor.Get()); - ActiveCommands.Add(Cmd); + ActiveCommands.Add(ToolCallId, Cmd); Cmd->OnCommandConfirmed.AddDynamic(this, &UAIBaseManager::CommandConfirmed); Cmd->OnCommandDone.AddDynamic(this, &UAIBaseManager::CommandFinished); @@ -337,28 +355,38 @@ void UAIBaseManager::RunMCPCommand(FString CommandName, FString Payload, FString Cmd->InitMCPCommand(World); Cmd->Execute(World, Payload); - return; } if (MCPManager && MCPManager->HasCommand(CommandName)) { - if (!ToolCallId.IsEmpty()) - MCPToolCallIds.Add(CommandName, ToolCallId); - MCPManager->ExecuteCommand(CommandName, Payload); + ActiveMCPCallIds.Add(ToolCallId); + MCPManager->ExecuteCommand(CommandName, Payload, ToolCallId); } + OnAIToolcallStarted.Broadcast(CommandName, ToolCallId, Payload); } void UAIBaseManager::ClearMCPCommand() { - for (UMCPUnrealCommand* Command : ActiveCommands) + for (auto& Pair : ActiveCommands) { - if (Command) + if (Pair.Value) { - Command->OnCommandDone.Clear(); - Command->OnCommandFailed.Clear(); + Pair.Value->OnCommandDone.Clear(); + Pair.Value->OnCommandFailed.Clear(); } } ActiveCommands.Empty(); + ActiveMCPCallIds.Empty(); +} + +bool UAIBaseManager::AnyToolcallRunning() const +{ + return ActiveCommands.Num() > 0 || ActiveMCPCallIds.Num() > 0; +} + +int UAIBaseManager::HowManyToolcallRunning() const +{ + return ActiveCommands.Num() + ActiveMCPCallIds.Num(); } FString UAIBaseManager::GetRoleAsString(EAvatarCoreAIPromptRole Role) @@ -384,13 +412,12 @@ void UAIBaseManager::CommandConfirmed(const FAIMessage& Message) void UAIBaseManager::CommandFinished(const FAIMessage& Message) { - ActiveCommands.RemoveAll([&Message](UMCPUnrealCommand* Cmd) - { - return Cmd && (Message.Id.IsEmpty() || Cmd->Id.Equals(Message.Id)); - }); + UMCPUnrealCommand* Cmd = ActiveCommands.FindRef(Message.Id); + FString ToolName = Cmd ? Cmd->GetCommandName() : TEXT(""); + ActiveCommands.Remove(Message.Id); + OnAIToolcallFinished.Broadcast(ToolName, Message.Id, Message.Message); SetNewState(EAvatarCoreAIState::Ready); - functionCallRunning = false; if (bDebugMode) BroadcastAILog(FString::Printf(TEXT("Command ran successfully. Answer: %s"), *Message.Message), true); else @@ -412,29 +439,21 @@ void UAIBaseManager::CommandFinished(const FAIMessage& Message) void UAIBaseManager::CommandFailed(const FAIMessage& Message) { - ActiveCommands.RemoveAll([&Message](UMCPUnrealCommand* Cmd) - { - return Cmd && (Message.Id.IsEmpty() || Cmd->Id.Equals(Message.Id)); - }); + UMCPUnrealCommand* Cmd = ActiveCommands.FindRef(Message.Id); + FString ToolName = Cmd ? Cmd->GetCommandName() : TEXT(""); + ActiveCommands.Remove(Message.Id); - functionCallRunning = false; + OnAIToolcallError.Broadcast(ToolName, Message.Id, Message.Message); SetNewState(EAvatarCoreAIState::Ready); BroadcastAILog(FString::Printf(TEXT("Command failed. Sending: %s"), *Message.Message), true); SendResponse(Message, false); } -void UAIBaseManager::MCPCommandFinished(const FString& Command, const FString& Payload) +void UAIBaseManager::MCPCommandFinished(const FString& Command, const FString& Payload, const FString& ToolCallId) { - FString FoundId; - FString* MCPId = MCPToolCallIds.Find(Command); - if (MCPId) - { - FoundId = *MCPId; - MCPToolCallIds.Remove(Command); - } - + ActiveMCPCallIds.Remove(ToolCallId); + OnAIToolcallFinished.Broadcast(Command, ToolCallId, Payload); SetNewState(EAvatarCoreAIState::Ready); - functionCallRunning = false; if (bDebugMode) BroadcastAILog(FString::Printf(TEXT("MCP Command '%s' ran successfully. Answer: %s"), *Command, *Payload), true); else @@ -442,10 +461,10 @@ void UAIBaseManager::MCPCommandFinished(const FString& Command, const FString& P FAIMessage FinishedCommandMessage; FinishedCommandMessage.Message = Payload; - if (!FoundId.IsEmpty()) + if (!ToolCallId.IsEmpty()) { FinishedCommandMessage.Role = EAvatarCoreAIPromptRole::Tool; - FinishedCommandMessage.Id = FoundId; + FinishedCommandMessage.Id = ToolCallId; } else { @@ -454,9 +473,10 @@ void UAIBaseManager::MCPCommandFinished(const FString& Command, const FString& P SendResponse(FinishedCommandMessage, false); } -void UAIBaseManager::MCPCommandFailed(const FString& Command, const FString& Payload) +void UAIBaseManager::MCPCommandFailed(const FString& Command, const FString& Payload, const FString& ToolCallId) { - functionCallRunning = false; + ActiveMCPCallIds.Remove(ToolCallId); + OnAIToolcallError.Broadcast(Command, ToolCallId, Payload); SetNewState(EAvatarCoreAIState::Ready); BroadcastAILog(FString::Printf(TEXT("MCP Command '%s' failed. Sending: %s"), *Command, *Payload), true); FAIMessage FailedCommandMessage; diff --git a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/MCP/FastMCP/FastMCPManager.cpp b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/MCP/FastMCP/FastMCPManager.cpp index 1615eb6..175c4a6 100644 --- a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/MCP/FastMCP/FastMCPManager.cpp +++ b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/MCP/FastMCP/FastMCPManager.cpp @@ -46,14 +46,31 @@ void UFastMCPManager::InitMCPManager(UMCPBaseConfig* InMCPConfig, bool DebugMode return; } + bool bKeepAlive = false; + #if WITH_EDITOR - bSpawnServerWhenNeeded = true; - CheckServerHealth(); + if (FastMCPConfig->FastMCPSettings.KeepAliveRule == EKeepAliveRule::Never) + bKeepAlive = false; + else + bKeepAlive = true; #else - StopFastMCPServer(); - StartFastMCPServer(); + if (FastMCPConfig->FastMCPSettings.KeepAliveRule == EKeepAliveRule::Always) + bKeepAlive = true; + else + bKeepAlive = false; #endif + if(bKeepAlive) + { + bSpawnServerWhenNeeded = true; + CheckServerHealth(); + } + else + { + StopFastMCPServer(); + StartFastMCPServer(); + } + } void UFastMCPManager::DeinitMCPManager() @@ -83,7 +100,7 @@ void UFastMCPManager::StartFastMCPServer() LogMessage(TEXT("Starting FastMCP Server...")); // Build command line arguments - FString Arguments = FString::Printf(TEXT("\"%s\" \"%s\""), *FastMCPConfig->PythonPath, *FastMCPConfig->Arguments); + FString Arguments = FString::Printf(TEXT("\"%s\" \"%s\""), *FastMCPConfig->FastMCPSettings.PythonPath, *FastMCPConfig->FastMCPSettings.Arguments); uint32 ProcId = 0; @@ -212,7 +229,7 @@ void UFastMCPManager::CheckServerHealth() { HealthCheckAttempts++; - if (HealthCheckAttempts > MaxHealthCheckAttempts) + if (HealthCheckAttempts > FastMCPConfig->FastMCPSettings.MaxHealthCheckTimeInSec) { LogError(TEXT("FastMCP Server failed to respond after maximum attempts"), EMCPManagerError::InitializationError); if (UWorld* World = GEngine->GetWorldFromContextObject(this, EGetWorldErrorMode::LogAndReturnNull)) @@ -409,7 +426,7 @@ bool UFastMCPManager::ParseToolsFromResponse(const FString& JsonResponse, TArray return false; } -void UFastMCPManager::ExecuteCommand(const FString& Command, const FString& Payload) +void UFastMCPManager::ExecuteCommand(const FString& Command, const FString& Payload, const FString& ToolCallId) { if (!bServerRunning) { @@ -420,6 +437,7 @@ void UFastMCPManager::ExecuteCommand(const FString& Command, const FString& Payl // Store current command for response handling CurrentExecutingCommand = Command; CurrentExecutingPayload = Payload; + CurrentToolCallId = ToolCallId; SetState(EMCPManagerState::Busy); LogMessage(FString::Printf(TEXT("Executing command: %s"), *Command)); @@ -428,7 +446,7 @@ void UFastMCPManager::ExecuteCommand(const FString& Command, const FString& Payl TSharedPtr CallMessage = MakeShareable(new FJsonObject); CallMessage->SetStringField(TEXT("jsonrpc"), TEXT("2.0")); CallMessage->SetStringField(TEXT("method"), TEXT("tools/call")); - CallMessage->SetNumberField(TEXT("id"), 3); + CallMessage->SetStringField(TEXT("id"), ToolCallId); TSharedPtr Params = MakeShareable(new FJsonObject); Params->SetStringField(TEXT("name"), Command); @@ -473,14 +491,14 @@ void UFastMCPManager::OnCommandExecuted(FHttpRequestPtr Request, FHttpResponsePt if (!bWasSuccessful || !Response.IsValid()) { LogError(FString::Printf(TEXT("Failed to execute command: %s"), *CurrentExecutingCommand), EMCPManagerError::ToolError); - OnMCPCommandFailed.Broadcast(CurrentExecutingCommand, CurrentExecutingPayload); + OnMCPCommandFailed.Broadcast(CurrentExecutingCommand, CurrentExecutingPayload, CurrentToolCallId); return; } if (Response->GetResponseCode() != 200) { LogError(FString::Printf(TEXT("Command execution failed with code %d: %s"), Response->GetResponseCode(), *CurrentExecutingCommand), EMCPManagerError::ToolError); - OnMCPCommandFailed.Broadcast(CurrentExecutingCommand, CurrentExecutingPayload); + OnMCPCommandFailed.Broadcast(CurrentExecutingCommand, CurrentExecutingPayload, CurrentToolCallId); return; } @@ -499,7 +517,7 @@ void UFastMCPManager::OnCommandExecuted(FHttpRequestPtr Request, FHttpResponsePt if ((*ResultObject)->HasField(TEXT("isError")) && (*ResultObject)->GetBoolField(TEXT("isError"))) { LogError(FString::Printf(TEXT("Command execution failed: %s - %s"), *CurrentExecutingCommand, *ResponseContent), EMCPManagerError::ToolError); - OnMCPCommandFailed.Broadcast(CurrentExecutingCommand, ResponseContent); + OnMCPCommandFailed.Broadcast(CurrentExecutingCommand, ResponseContent, CurrentToolCallId); return; } } @@ -508,7 +526,7 @@ void UFastMCPManager::OnCommandExecuted(FHttpRequestPtr Request, FHttpResponsePt // Otherwise, broadcast success event with response LogMessage(FString::Printf(TEXT("Command executed successfully: %s"), *CurrentExecutingCommand)); - OnMCPCommandDone.Broadcast(CurrentExecutingCommand, ResponseContent); + OnMCPCommandDone.Broadcast(CurrentExecutingCommand, ResponseContent, CurrentToolCallId); } void UFastMCPManager::SendMCPMessage(const FString& JsonMessage, TFunction OnComplete) diff --git a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/MCP/MCPUnrealCommand.cpp b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/MCP/MCPUnrealCommand.cpp index 0465a1d..a786207 100644 --- a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/MCP/MCPUnrealCommand.cpp +++ b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/MCP/MCPUnrealCommand.cpp @@ -5,6 +5,7 @@ void UMCPUnrealCommand::Execute_Implementation(UWorld* WorldReference, const FString& Payload) { + SetWorldContext(WorldReference); StartTimeout(); } @@ -114,9 +115,26 @@ UObject* UMCPUnrealCommand::GetWorldContextObject() const UWorld* UMCPUnrealCommand::GetWorld() const { - if (RequiredWorldContext) - return RequiredWorldContext->GetWorld(); - if (const UObject* Outer = GetOuter()) - return Outer->GetWorld(); - return nullptr; +#if WITH_EDITOR + // Optional editor-only fallback before the game starts. This keeps the warning flood out of the way. + if (GEditor && HasAnyFlags(RF_ClassDefaultObject)) + { + const FWorldContext& EditorWorldContext = GEditor->GetEditorWorldContext(false); + return EditorWorldContext.World(); + } + + // If it is not a ClassDefaultObject and still world is null, throw an error. + if (!RequiredWorldContext && !WarningMessageShown) { + FMessageLog("PIE").Error(FText::Format( + NSLOCTEXT("ObjectWithContext", "MissingWorldContext", + "WorldContextObject is not set on object '{0}' of class '{1}'."), + FText::FromString(GetName()), + FText::FromString(GetNameSafe(GetClass())) + )); + FMessageLog("PIE").Notify(); + WarningMessageShown = true; + } +#endif + + return RequiredWorldContext ? RequiredWorldContext->GetWorld() : nullptr; } diff --git a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/RealtimeAPI/AvatarCoreAIRealtime.cpp b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/RealtimeAPI/AvatarCoreAIRealtime.cpp index 654ac67..fa8d7d5 100644 --- a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/RealtimeAPI/AvatarCoreAIRealtime.cpp +++ b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/RealtimeAPI/AvatarCoreAIRealtime.cpp @@ -545,12 +545,12 @@ void UAvatarCoreAIRealtime::OnWebSocketConnectionStringReceived(const FString& M else if (RequestItem == EOpenAIRequestItem::audio_transcript || RequestItem == EOpenAIRequestItem::text) ResponseTextDone = true; - if (functionCallRunning) { + if (AnyToolcallRunning()) { return; } //float CurrentRequestDuration = (FDateTime::Now() - CurrentRequestStartTime).GetTotalSeconds(); - if (CurrentRequestDuration < 0.25f && !functionCallRunning && !CurrentRequestID.IsEmpty()) + if (CurrentRequestDuration < 0.25f && !AnyToolcallRunning() && !CurrentRequestID.IsEmpty()) { CurrentRequestID.Empty(); if (CurrentRetries < MaxRetries) diff --git a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/Tools/OpenAiModerator.cpp b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/Tools/OpenAiModerator.cpp deleted file mode 100644 index a206d76..0000000 --- a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/Tools/OpenAiModerator.cpp +++ /dev/null @@ -1,178 +0,0 @@ -// Fill out your copyright notice in the Description page of Project Settings. - - -#include "Tools/OpenAiModerator.h" - -#include "HttpModule.h" -#include "Interfaces/IHttpResponse.h" -#include "Dom/JsonObject.h" -#include "Serialization/JsonReader.h" -#include "Serialization/JsonSerializer.h" - -FString UOpenAiModerator::StoredApiKey; -FString UOpenAiModerator::StoredModel = TEXT("omni-moderation-latest"); -TArray> UOpenAiModerator::ActiveRequests; - -void UOpenAiModerator::InitModeration(const FString& ApiKey, const FString& Model) -{ - StoredApiKey = ApiKey; - StoredModel = Model; -} - -UOpenAiModerator* UOpenAiModerator::UseModeration(UObject* WorldContextObject, const FString& Text) -{ - UOpenAiModerator* Action = NewObject(); - Action->InputText = Text; - Action->RegisterWithGameInstance(WorldContextObject); - return Action; -} - -void UOpenAiModerator::Activate() -{ - if (StoredApiKey.IsEmpty()) - { - UE_LOG(LogTemp, Error, TEXT("OpenAiModerator: API key not set. Call InitModeration first.")); - FModerationResult EmptyResult; - Passed.Broadcast(EmptyResult); - return; - } - - RequestStartTime = FPlatformTime::Seconds(); - ActiveRequests.Add(this); - - TSharedRef HttpRequest = FHttpModule::Get().CreateRequest(); - HttpRequest->SetURL(TEXT("https://api.openai.com/v1/moderations")); - HttpRequest->SetVerb(TEXT("POST")); - HttpRequest->SetHeader(TEXT("Content-Type"), TEXT("application/json")); - HttpRequest->SetHeader(TEXT("Authorization"), FString::Printf(TEXT("Bearer %s"), *StoredApiKey)); - - const FString Body = FString::Printf(TEXT("{\"model\":\"%s\",\"input\":\"%s\"}"), - *StoredModel, - *InputText.ReplaceCharWithEscapedChar()); - - HttpRequest->SetContentAsString(Body); - - HttpRequest->OnProcessRequestComplete().BindUObject(this, &UOpenAiModerator::HandleHttpResponse); - - if (!HttpRequest->ProcessRequest()) - { - UE_LOG(LogTemp, Error, TEXT("OpenAiModerator: Failed to start HTTP request.")); - FModerationResult EmptyResult; - Passed.Broadcast(EmptyResult); - } -} - -void UOpenAiModerator::ClearModeration() -{ - TArray> RequestsCopy = ActiveRequests; - ActiveRequests.Empty(); - - for (const TWeakObjectPtr& Weak : RequestsCopy) - { - if (UOpenAiModerator* Action = Weak.Get()) - { - Action->SetReadyToDestroy(); - } - } -} - -void UOpenAiModerator::HandleHttpResponse(FHttpRequestPtr Request, FHttpResponsePtr Response, bool bWasSuccessful) -{ - ActiveRequests.RemoveAll([this](const TWeakObjectPtr& Weak) { return !Weak.IsValid() || Weak.Get() == this; }); - - const float ElapsedTime = static_cast(FPlatformTime::Seconds() - RequestStartTime); - - FModerationResult Result; - Result.ExecutionTime = ElapsedTime; - - if (!bWasSuccessful || !Response.IsValid()) - { - UE_LOG(LogTemp, Error, TEXT("OpenAiModerator: HTTP request failed.")); - Passed.Broadcast(Result); - return; - } - - const int32 Code = Response->GetResponseCode(); - const FString Body = Response->GetContentAsString(); - - if (Code < 200 || Code >= 300) - { - UE_LOG(LogTemp, Error, TEXT("OpenAiModerator: HTTP %d — %s"), Code, *Body); - Passed.Broadcast(Result); - return; - } - - TSharedPtr RootObject; - TSharedRef> Reader = TJsonReaderFactory<>::Create(Body); - if (!FJsonSerializer::Deserialize(Reader, RootObject) || !RootObject.IsValid()) - { - UE_LOG(LogTemp, Error, TEXT("OpenAiModerator: Failed to parse response JSON.")); - Passed.Broadcast(Result); - return; - } - - const TArray>* ResultsArray; - if (!RootObject->TryGetArrayField(TEXT("results"), ResultsArray) || ResultsArray->Num() == 0) - { - UE_LOG(LogTemp, Error, TEXT("OpenAiModerator: No results in response.")); - Passed.Broadcast(Result); - return; - } - - const TSharedPtr& FirstResult = (*ResultsArray)[0]->AsObject(); - if (!FirstResult.IsValid()) - { - UE_LOG(LogTemp, Error, TEXT("OpenAiModerator: Invalid result object.")); - Passed.Broadcast(Result); - return; - } - - Result.bFlagged = FirstResult->GetBoolField(TEXT("flagged")); - - const TSharedPtr* CategoriesObject; - if (FirstResult->TryGetObjectField(TEXT("categories"), CategoriesObject)) - { - FModerationCategories& C = Result.Categories; - C.bSexual = (*CategoriesObject)->GetBoolField(TEXT("sexual")); - C.bSexualMinors = (*CategoriesObject)->GetBoolField(TEXT("sexual/minors")); - C.bHarassment = (*CategoriesObject)->GetBoolField(TEXT("harassment")); - C.bHarassmentThreatening = (*CategoriesObject)->GetBoolField(TEXT("harassment/threatening")); - C.bHate = (*CategoriesObject)->GetBoolField(TEXT("hate")); - C.bHateThreatening = (*CategoriesObject)->GetBoolField(TEXT("hate/threatening")); - C.bIllicit = (*CategoriesObject)->GetBoolField(TEXT("illicit")); - C.bIllicitViolent = (*CategoriesObject)->GetBoolField(TEXT("illicit/violent")); - C.bSelfHarm = (*CategoriesObject)->GetBoolField(TEXT("self-harm")); - C.bSelfHarmIntent = (*CategoriesObject)->GetBoolField(TEXT("self-harm/intent")); - C.bSelfHarmInstructions = (*CategoriesObject)->GetBoolField(TEXT("self-harm/instructions")); - C.bViolence = (*CategoriesObject)->GetBoolField(TEXT("violence")); - C.bViolenceGraphic = (*CategoriesObject)->GetBoolField(TEXT("violence/graphic")); - } - - const TSharedPtr* ScoresObject; - if (FirstResult->TryGetObjectField(TEXT("category_scores"), ScoresObject)) - { - FModerationCategoryScores& S = Result.CategoryScores; - S.Sexual = (*ScoresObject)->GetNumberField(TEXT("sexual")); - S.SexualMinors = (*ScoresObject)->GetNumberField(TEXT("sexual/minors")); - S.Harassment = (*ScoresObject)->GetNumberField(TEXT("harassment")); - S.HarassmentThreatening = (*ScoresObject)->GetNumberField(TEXT("harassment/threatening")); - S.Hate = (*ScoresObject)->GetNumberField(TEXT("hate")); - S.HateThreatening = (*ScoresObject)->GetNumberField(TEXT("hate/threatening")); - S.Illicit = (*ScoresObject)->GetNumberField(TEXT("illicit")); - S.IllicitViolent = (*ScoresObject)->GetNumberField(TEXT("illicit/violent")); - S.SelfHarm = (*ScoresObject)->GetNumberField(TEXT("self-harm")); - S.SelfHarmIntent = (*ScoresObject)->GetNumberField(TEXT("self-harm/intent")); - S.SelfHarmInstructions = (*ScoresObject)->GetNumberField(TEXT("self-harm/instructions")); - S.Violence = (*ScoresObject)->GetNumberField(TEXT("violence")); - S.ViolenceGraphic = (*ScoresObject)->GetNumberField(TEXT("violence/graphic")); - } - - if (Result.bFlagged) - { - Flagged.Broadcast(Result); - } - else - { - Passed.Broadcast(Result); - } -} diff --git a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/Tools/OpenAiResponder.cpp b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/Tools/OpenAiResponder.cpp index a58d7e8..971cada 100644 --- a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/Tools/OpenAiResponder.cpp +++ b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/Tools/OpenAiResponder.cpp @@ -12,12 +12,14 @@ FString UOpenAiResponder::StoredApiKey; FString UOpenAiResponder::StoredModel = TEXT("gpt-4.1"); +bool UOpenAiResponder::IsResponderReady = false; TArray> UOpenAiResponder::ActiveRequests; void UOpenAiResponder::InitResponder(const FString& ApiKey, const FString& Model) { StoredApiKey = ApiKey; StoredModel = Model; + IsResponderReady = true; } UOpenAiResponder* UOpenAiResponder::CallOpenAiResponse( @@ -44,6 +46,11 @@ UOpenAiResponder* UOpenAiResponder::CallOpenAiResponse( return Action; } +bool UOpenAiResponder::IsOpenAIResponderReady() +{ + return IsResponderReady; +} + void UOpenAiResponder::Activate() { if (StoredApiKey.IsEmpty()) diff --git a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/Tools/OpenRouterResponder.cpp b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/Tools/OpenRouterResponder.cpp index 2baa963..ad06d1f 100644 --- a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/Tools/OpenRouterResponder.cpp +++ b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/Tools/OpenRouterResponder.cpp @@ -15,6 +15,7 @@ FString UOpenRouterResponder::StoredModel = TEXT("openai/gpt-4.1"); FString UOpenRouterResponder::StoredSiteUrl; FString UOpenRouterResponder::StoredSiteName; TArray> UOpenRouterResponder::ActiveRequests; +bool UOpenRouterResponder::IsResponderReady = false; void UOpenRouterResponder::InitResponder(const FString& ApiKey, const FString& Model, const FString& SiteUrl, const FString& SiteName) { @@ -22,6 +23,7 @@ void UOpenRouterResponder::InitResponder(const FString& ApiKey, const FString& M StoredModel = Model; StoredSiteUrl = SiteUrl; StoredSiteName = SiteName; + IsResponderReady = true; } UOpenRouterResponder* UOpenRouterResponder::CallOpenRouterResponse( @@ -46,6 +48,11 @@ UOpenRouterResponder* UOpenRouterResponder::CallOpenRouterResponse( return Action; } +bool UOpenRouterResponder::IsOpenRouterResponderReady() +{ + return IsResponderReady; +} + void UOpenRouterResponder::Activate() { if (StoredApiKey.IsEmpty()) diff --git a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/AIBaseConfig.h b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/AIBaseConfig.h index 5d19223..a2d96b6 100644 --- a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/AIBaseConfig.h +++ b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/AIBaseConfig.h @@ -29,10 +29,6 @@ struct FGlobalAISettings { GENERATED_BODY() - // Check user transcription for inappropriate behaviour first (adds a delay!) - UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|MCP", meta = (ExposeOnSpawn = "true")) - bool bUseModeration = false; - // Boot up a FastMCP Server on Startup (On close keep open in Editor Mode, kill in shipping) UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|MCP", meta = (ExposeOnSpawn = "true")) bool bUseMCPServer = false; diff --git a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/AIBaseManager.h b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/AIBaseManager.h index 9293bd3..b903491 100644 --- a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/AIBaseManager.h +++ b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/AIBaseManager.h @@ -6,6 +6,7 @@ #include "UObject/Object.h" #include "AIBaseConfig.h" #include "AvatarCoreAIEnumsAndStructs.h" +#include "AvatarCoreSharedEnums.h" #include "MCP/MCPBaseManager.h" #include "AIBaseManager.generated.h" @@ -19,6 +20,9 @@ DECLARE_DYNAMIC_MULTICAST_DELEGATE_TwoParams(FOnAIError, FString, ErrorMessage, DECLARE_DYNAMIC_MULTICAST_DELEGATE_TwoParams(FMulticastDelegateRealtimeAPIAudioChunk, const TArray, PCMData, bool, IsFinal); DECLARE_DYNAMIC_MULTICAST_DELEGATE(FOnAIDelayedAnswer); DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FOnAIActivationStateChanged, bool, IsActive); +DECLARE_DYNAMIC_MULTICAST_DELEGATE_ThreeParams(FOnAIToolcallStarted, FString, ToolName, FString, ToolCallId, FString, Payload); +DECLARE_DYNAMIC_MULTICAST_DELEGATE_ThreeParams(FOnAIToolcallFinished, FString, ToolName, FString, ToolCallId, FString, Result); +DECLARE_DYNAMIC_MULTICAST_DELEGATE_ThreeParams(FOnAIToolcallError, FString, ToolName, FString, ToolCallId, FString, ErrorMessage); class UMCPManager; @@ -57,6 +61,18 @@ public: UPROPERTY(BlueprintAssignable, Category = "AvatarCoreAI|Events") FOnAIActivationStateChanged OnAIActivationStateChanged; + // Fired when a tool/function call is dispatched (Payload is raw JSON input) + UPROPERTY(BlueprintAssignable, Category = "AvatarCoreAI|Events") + FOnAIToolcallStarted OnAIToolcallStarted; + + // Fired when a tool/function call completes successfully (Result is raw JSON output) + UPROPERTY(BlueprintAssignable, Category = "AvatarCoreAI|Events") + FOnAIToolcallFinished OnAIToolcallFinished; + + // Fired when a tool/function call fails + UPROPERTY(BlueprintAssignable, Category = "AvatarCoreAI|Events") + FOnAIToolcallError OnAIToolcallError; + /** * Initializes the AI Manager with the given config and adds this UObject to the root set. @@ -67,6 +83,9 @@ public: UFUNCTION(BlueprintCallable, Category = "AvatarCoreAI") virtual void InitAIManagerChild(UAIBaseConfig* AIConfig, AActor* InWorldReferenceActor); + UFUNCTION(BlueprintCallable, Category = "AvatarCoreAI") + void SetAILanguage(ELanguage NewLanguage); + /** Returns the actor used as world context for commands */ UFUNCTION(BlueprintPure, Category = "AvatarCoreAI") AActor* GetWorldReferenceActor() const { return WorldReferenceActor.Get(); } @@ -196,6 +215,14 @@ public: UFUNCTION() void OnRequestTimeout(); + // Returns true if any Unreal command instance is active or a function call is in progress + UFUNCTION(BlueprintPure, Category = "AvatarCoreAI") + bool AnyToolcallRunning() const; + + // Returns amount of toolcalls (Unreal Command and MCP) that are currenlty active + UFUNCTION(BlueprintPure, Category = "AvatarCoreAI") + int HowManyToolcallRunning() const; + // Add a command at runtime (handles AddToRoot) UFUNCTION(BlueprintCallable, Category = "AvatarCoreAI|MCP") UMCPBaseManager* GetMCPManager(); @@ -242,11 +269,11 @@ protected: /** Bound to MCPManager::OnMCPCommandDone — constructs FAIMessage from raw strings. */ UFUNCTION() - void MCPCommandFinished(const FString& Command, const FString& Payload); + void MCPCommandFinished(const FString& Command, const FString& Payload, const FString& ToolCallId); /** Bound to MCPManager::OnMCPCommandFailed. */ UFUNCTION() - void MCPCommandFailed(const FString& Command, const FString& Payload); + void MCPCommandFailed(const FString& Command, const FString& Payload, const FString& ToolCallId); //Add System/User/Assistant Message to memory archive void AddMessageToArray(FAIMessage NewMessage); @@ -285,11 +312,10 @@ protected: TArray> UnrealCommandClasses; TArray UnrealCommandsToolInfos; - // Maps MCP server command name → tool_call_id for propagation through MCPCommandFinished - TMap MCPToolCallIds; - UPROPERTY() - TArray ActiveCommands; + TMap ActiveCommands; // ToolCallId → Unreal command + + TSet ActiveMCPCallIds; // ToolCallIds of in-flight MCP server commands /** MCP Manager for FastMCP server communication */ UPROPERTY() @@ -310,9 +336,6 @@ protected: //Cached Answer int ResponseID = 0; - //There is a function call in progress - bool functionCallRunning = false; - //Current State the AI Manager EAvatarCoreAIState CurrentAIState = EAvatarCoreAIState::Disconnected; @@ -329,4 +352,6 @@ private: TQueue ResponseQueue; + ELanguage ForcedLanguage = ELanguage::NONE; + }; \ No newline at end of file diff --git a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/MCP/FastMCP/FastMCPConfig.h b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/MCP/FastMCP/FastMCPConfig.h index a9754e2..4bdb793 100644 --- a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/MCP/FastMCP/FastMCPConfig.h +++ b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/MCP/FastMCP/FastMCPConfig.h @@ -4,8 +4,31 @@ #include "CoreMinimal.h" #include "MCP/MCPBaseConfig.h" +#include "AvatarCoreSharedEnums.h" #include "FastMCPConfig.generated.h" + +USTRUCT(BlueprintType) +struct FFastMCPSettings +{ + GENERATED_BODY() + + //Custom python environment - "python" will use the system default + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|MCP", meta = (ExposeOnSpawn = "true")) + FString PythonPath = "python"; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|MCP", meta = (ExposeOnSpawn = "true")) + EKeepAliveRule KeepAliveRule = EKeepAliveRule::EditorOnly; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|MCP", meta = (ExposeOnSpawn = "true")) + int MaxHealthCheckTimeInSec = 180; + + //Additional arguments to pass + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|MCP", meta = (ExposeOnSpawn = "true")) + FString Arguments = ""; + +}; + /** * */ @@ -18,14 +41,9 @@ public: //Direction to the Script that start FastMCP UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|MCP", meta = (ExposeOnSpawn = "true")) - FString MCPExecutable = FPaths::ProjectContentDir() +"DB/FastMCP/FastMCPServer.bat"; + FString MCPExecutable = FPaths::ProjectContentDir() + "DB/FastMCP/FastMCPServer.bat"; - //Custom python environment - "python" will use the system default - UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreTTS|CoquiTTS", meta = (ExposeOnSpawn = "true")) - FString PythonPath = "python"; - - //Additional arguments to pass UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|MCP", meta = (ExposeOnSpawn = "true")) - FString Arguments = ""; + FFastMCPSettings FastMCPSettings; }; diff --git a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/MCP/FastMCP/FastMCPManager.h b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/MCP/FastMCP/FastMCPManager.h index b24e5aa..07b468e 100644 --- a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/MCP/FastMCP/FastMCPManager.h +++ b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/MCP/FastMCP/FastMCPManager.h @@ -24,7 +24,7 @@ public: virtual void InitMCPManager(UMCPBaseConfig* InMCPConfig, bool DebugMode) override; virtual void DeinitMCPManager() override; virtual void FetchAvailableTools() override; - virtual void ExecuteCommand(const FString& Command, const FString& Payload) override; + virtual void ExecuteCommand(const FString& Command, const FString& Payload, const FString& ToolCallId) override; public: // Helper to crop to first JSON object in a string (for SSE or noisy responses) @@ -67,9 +67,9 @@ private: // Health check timer FTimerHandle HealthCheckTimer; int32 HealthCheckAttempts; - static constexpr int32 MaxHealthCheckAttempts = 90; // 90 seconds max wait // Current command tracking FString CurrentExecutingCommand; FString CurrentExecutingPayload; + FString CurrentToolCallId; }; diff --git a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/MCP/MCPBaseManager.h b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/MCP/MCPBaseManager.h index f73df21..43e405b 100644 --- a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/MCP/MCPBaseManager.h +++ b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/MCP/MCPBaseManager.h @@ -32,8 +32,8 @@ DECLARE_DYNAMIC_MULTICAST_DELEGATE_TwoParams(FOnMCPManagerError, const FString&, DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FOnMCPManagerStateChanged, EMCPManagerState, NewMCPManagerState); DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FOnMCPLog, const FString&, LogMessage); DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FOnMCPToolsUpdated, const TArray&, AvailableTools); -DECLARE_DYNAMIC_MULTICAST_DELEGATE_TwoParams(FOnMCPCommandDone, const FString&, Command, const FString&, Payload); -DECLARE_DYNAMIC_MULTICAST_DELEGATE_TwoParams(FOnMCPCommandFailed, const FString&, Command, const FString&, Payload); +DECLARE_DYNAMIC_MULTICAST_DELEGATE_ThreeParams(FOnMCPCommandDone, const FString&, Command, const FString&, Payload, const FString&, ToolCallId); +DECLARE_DYNAMIC_MULTICAST_DELEGATE_ThreeParams(FOnMCPCommandFailed, const FString&, Command, const FString&, Payload, const FString&, ToolCallId); UCLASS(Abstract, Blueprintable, BlueprintType) class AVATARCORE_AI_API UMCPBaseManager : public UObject @@ -81,7 +81,7 @@ public: bool HasCommand(const FString& Command); UFUNCTION(BlueprintCallable, Category = "AvatarCoreAI|MCP|Operations") - virtual void ExecuteCommand(const FString& Command, const FString& Payload) {}; + virtual void ExecuteCommand(const FString& Command, const FString& Payload, const FString& ToolCallId) {}; // Blueprint Functions - State Management UFUNCTION(BlueprintCallable, BlueprintPure, Category = "AvatarCoreAI|MCP|State") diff --git a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/MCP/MCPUnrealCommand.h b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/MCP/MCPUnrealCommand.h index a2480d8..33e8cac 100644 --- a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/MCP/MCPUnrealCommand.h +++ b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/MCP/MCPUnrealCommand.h @@ -37,6 +37,8 @@ public: UFUNCTION(BlueprintCallable, Category = "Context") UObject* GetWorldContextObject() const; + virtual UWorld* GetWorld() const override; + // Result event (success) UPROPERTY(BlueprintAssignable, Category = "Command") FOnAICommandDone OnCommandDone; @@ -103,8 +105,9 @@ protected: UFUNCTION(BlueprintCallable, Category = "Command") AActor* GetActorOfClass(UWorld* World, TSubclassOf ActorClass) const; - - virtual UWorld* GetWorld() const override; void StartTimeout(); void OnTimeout(); + +private: + mutable bool WarningMessageShown; }; diff --git a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/Tools/OpenAiModerator.h b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/Tools/OpenAiModerator.h deleted file mode 100644 index 619235d..0000000 --- a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/Tools/OpenAiModerator.h +++ /dev/null @@ -1,159 +0,0 @@ -// Fill out your copyright notice in the Description page of Project Settings. - -#pragma once - -#include "CoreMinimal.h" -#include "Kismet/BlueprintAsyncActionBase.h" -#include "Http.h" -#include "OpenAiModerator.generated.h" - -USTRUCT(BlueprintType) -struct FModerationCategories -{ - GENERATED_BODY() - - UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation") - bool bSexual = false; - - UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation") - bool bSexualMinors = false; - - UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation") - bool bHarassment = false; - - UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation") - bool bHarassmentThreatening = false; - - UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation") - bool bHate = false; - - UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation") - bool bHateThreatening = false; - - UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation") - bool bIllicit = false; - - UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation") - bool bIllicitViolent = false; - - UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation") - bool bSelfHarm = false; - - UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation") - bool bSelfHarmIntent = false; - - UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation") - bool bSelfHarmInstructions = false; - - UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation") - bool bViolence = false; - - UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation") - bool bViolenceGraphic = false; -}; - -USTRUCT(BlueprintType) -struct FModerationCategoryScores -{ - GENERATED_BODY() - - UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation") - float Sexual = 0.f; - - UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation") - float SexualMinors = 0.f; - - UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation") - float Harassment = 0.f; - - UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation") - float HarassmentThreatening = 0.f; - - UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation") - float Hate = 0.f; - - UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation") - float HateThreatening = 0.f; - - UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation") - float Illicit = 0.f; - - UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation") - float IllicitViolent = 0.f; - - UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation") - float SelfHarm = 0.f; - - UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation") - float SelfHarmIntent = 0.f; - - UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation") - float SelfHarmInstructions = 0.f; - - UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation") - float Violence = 0.f; - - UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation") - float ViolenceGraphic = 0.f; -}; - -USTRUCT(BlueprintType) -struct FModerationResult -{ - GENERATED_BODY() - - UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation") - bool bFlagged = false; - - UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation") - FModerationCategories Categories; - - UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation") - FModerationCategoryScores CategoryScores; - - UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation") - float ExecutionTime = 0.f; -}; - -DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FOnModerationResult, const FModerationResult&, Result); - -/** - * Async Blueprint node for OpenAI Moderation API. - * Call InitModeration once to set API key and model, then use UseModeration to classify text. - */ -UCLASS() -class AVATARCORE_AI_API UOpenAiModerator : public UBlueprintAsyncActionBase -{ - GENERATED_BODY() - -public: - /** Set API key and model. Call once before using UseModeration. */ - UFUNCTION(BlueprintCallable, Category = "AvatarCoreAI|Moderation", meta = (DisplayName = "Init Moderation")) - static void InitModeration(const FString& ApiKey, const FString& Model = TEXT("omni-moderation-latest")); - - /** Classify text using the OpenAI Moderation API. Returns via Flagged or Passed exec pins. */ - UFUNCTION(BlueprintCallable, Category = "AvatarCoreAI|Moderation", meta = (BlueprintInternalUseOnly = "true", DisplayName = "Use Moderation", WorldContext = "WorldContextObject")) - static UOpenAiModerator* UseModeration(UObject* WorldContextObject, const FString& Text); - - /** Cancel all active moderation requests. */ - UFUNCTION(BlueprintCallable, Category = "AvatarCoreAI|Moderation", meta = (DisplayName = "Clear Moderation")) - static void ClearModeration(); - - UPROPERTY(BlueprintAssignable) - FOnModerationResult Flagged; - - UPROPERTY(BlueprintAssignable) - FOnModerationResult Passed; - - virtual void Activate() override; - -private: - void HandleHttpResponse(FHttpRequestPtr Request, FHttpResponsePtr Response, bool bWasSuccessful); - - FString InputText; - double RequestStartTime = 0.0; - - static FString StoredApiKey; - static FString StoredModel; - static TArray> ActiveRequests; -}; diff --git a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/Tools/OpenAiResponder.h b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/Tools/OpenAiResponder.h index c3f4b80..7d0627d 100644 --- a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/Tools/OpenAiResponder.h +++ b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/Tools/OpenAiResponder.h @@ -123,6 +123,10 @@ public: bool bUseWebSearch = false, EResponderToolChoice ToolChoice = EResponderToolChoice::None); + /** Is responder ready and api key set? */ + UFUNCTION(BlueprintPure, Category = "AvatarCoreAI|Responder", meta = (DisplayName = "Is OpenAI Responder ready?")) + static bool IsOpenAIResponderReady(); + /** Cancel all active response requests. */ UFUNCTION(BlueprintCallable, Category = "AvatarCoreAI|Responder", meta = (DisplayName = "Clear All Responses")) static void ClearAllResponses(); @@ -148,6 +152,8 @@ private: EResponderToolChoice InputToolChoice = EResponderToolChoice::None; double RequestStartTime = 0.0; + static bool IsResponderReady; + static FString StoredApiKey; static FString StoredModel; static TArray> ActiveRequests; diff --git a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/Tools/OpenRouterResponder.h b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/Tools/OpenRouterResponder.h index 2937c7f..eab9cbd 100644 --- a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/Tools/OpenRouterResponder.h +++ b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/Tools/OpenRouterResponder.h @@ -104,6 +104,10 @@ public: ERouterReasoning Reasoning = ERouterReasoning::None, ERouterToolChoice ToolChoice = ERouterToolChoice::Auto); + /** Is responder ready and api key set? */ + UFUNCTION(BlueprintPure, Category = "AvatarCoreAI|Router", meta = (DisplayName = "Is OpenRouter Responder ready?")) + static bool IsOpenRouterResponderReady(); + /** Cancel all active OpenRouter requests. */ UFUNCTION(BlueprintCallable, Category = "AvatarCoreAI|Router", meta = (DisplayName = "Clear All Router Responses")) static void ClearAllResponses(); @@ -128,6 +132,8 @@ private: ERouterToolChoice InputToolChoice = ERouterToolChoice::Auto; double RequestStartTime = 0.0; + static bool IsResponderReady; + static FString StoredApiKey; static FString StoredModel; static FString StoredSiteUrl; diff --git a/Unreal/Plugins/AvatarCore_Manager/AvatarCore_Manager.uplugin b/Unreal/Plugins/AvatarCore_Manager/AvatarCore_Manager.uplugin index 6717e27..7fe439d 100644 --- a/Unreal/Plugins/AvatarCore_Manager/AvatarCore_Manager.uplugin +++ b/Unreal/Plugins/AvatarCore_Manager/AvatarCore_Manager.uplugin @@ -37,6 +37,10 @@ { "Name": "AvatarCore_TTS", "Enabled": true + }, + { + "Name": "AvatarCore_Shared", + "Enabled": true } ] } \ No newline at end of file diff --git a/Unreal/Plugins/AvatarCore_Manager/Content/AvatarCoreManager.uasset b/Unreal/Plugins/AvatarCore_Manager/Content/AvatarCoreManager.uasset index 5e84c0e..4759a8d 100644 --- a/Unreal/Plugins/AvatarCore_Manager/Content/AvatarCoreManager.uasset +++ b/Unreal/Plugins/AvatarCore_Manager/Content/AvatarCoreManager.uasset @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:1968895b296e8afec8db3b6546018f0c99ad8620470d7b913ea7d2dc5129891d -size 2130952 +oid sha256:175928fdf21fb09b80a66c1eebc8fc215f349897b7a8bce9c2b998c6e2e2dbfe +size 2087231 diff --git a/Unreal/Plugins/AvatarCore_Manager/Content/StateManagement/States/BP_BaseState.uasset b/Unreal/Plugins/AvatarCore_Manager/Content/StateManagement/States/BP_BaseState.uasset index 5d89426..815f6cb 100644 --- a/Unreal/Plugins/AvatarCore_Manager/Content/StateManagement/States/BP_BaseState.uasset +++ b/Unreal/Plugins/AvatarCore_Manager/Content/StateManagement/States/BP_BaseState.uasset @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:4bf4b1bdf42d4d3183dd20fc7202475b1defdf208801ed03092608b9ee0581cf -size 58074 +oid sha256:f7ffd5931e316b3744eed8f8a22070850ee5b3385a034838b59495666f3da792 +size 119538 diff --git a/Unreal/Plugins/AvatarCore_Manager/Content/UnrealCommands/AICommand_CurrentLocation.uasset b/Unreal/Plugins/AvatarCore_Manager/Content/UnrealCommands/AICommand_CurrentLocation.uasset index cfcc03a..c124a1f 100644 --- a/Unreal/Plugins/AvatarCore_Manager/Content/UnrealCommands/AICommand_CurrentLocation.uasset +++ b/Unreal/Plugins/AvatarCore_Manager/Content/UnrealCommands/AICommand_CurrentLocation.uasset @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:69fecb1290b5ea7f08b52a1f1b1334735e4b4adc13efaa49fc229301cfb09c5b -size 50670 +oid sha256:11626ab61f22fa125c2b1e5cf06226f83829795f1ecb9fed3efdeeedd33531b7 +size 51812 diff --git a/Unreal/Plugins/AvatarCore_Manager/Content/UnrealCommands/DebutWait_Command.uasset b/Unreal/Plugins/AvatarCore_Manager/Content/UnrealCommands/DebutWait_Command.uasset new file mode 100644 index 0000000..80c23c6 --- /dev/null +++ b/Unreal/Plugins/AvatarCore_Manager/Content/UnrealCommands/DebutWait_Command.uasset @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fd12e91bc373426fe0aafb4ce739bb8c807a92ca4b67bae408e67176303b37f7 +size 24269 diff --git a/Unreal/Plugins/AvatarCore_Manager/Content/Widgets/Debug/Pages/ChildWidget/W_AutoTranslator.uasset b/Unreal/Plugins/AvatarCore_Manager/Content/Widgets/Debug/Pages/ChildWidget/W_AutoTranslator.uasset index 44b12ee..32b0fbd 100644 --- a/Unreal/Plugins/AvatarCore_Manager/Content/Widgets/Debug/Pages/ChildWidget/W_AutoTranslator.uasset +++ b/Unreal/Plugins/AvatarCore_Manager/Content/Widgets/Debug/Pages/ChildWidget/W_AutoTranslator.uasset @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:8e7864d0268a6cbf7845c46ea6f92417b79824d535aae74b064bfb8c2320f9d3 -size 150156 +oid sha256:26d228f769140f161a6a3ac6c335134dcb8970e907a0a32277a9976247581cd6 +size 187818 diff --git a/Unreal/Plugins/AvatarCore_Manager/Content/Widgets/Debug/Pages/W_DebugAvatarCoreSTT.uasset b/Unreal/Plugins/AvatarCore_Manager/Content/Widgets/Debug/Pages/W_DebugAvatarCoreSTT.uasset index ad36bec..4050f5e 100644 --- a/Unreal/Plugins/AvatarCore_Manager/Content/Widgets/Debug/Pages/W_DebugAvatarCoreSTT.uasset +++ b/Unreal/Plugins/AvatarCore_Manager/Content/Widgets/Debug/Pages/W_DebugAvatarCoreSTT.uasset @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:772038ed7c65b69d974467ada2114271d5649786cd18a5e6fec92f3591e444eb -size 419134 +oid sha256:42b42fa9a6c60c2c1cc46e0e67a51b9b1ba58c78088cafa18234a27f48b3fc8f +size 446140 diff --git a/Unreal/Plugins/AvatarCore_Manager/Source/AvatarCore_Manager/AvatarCore_Manager.Build.cs b/Unreal/Plugins/AvatarCore_Manager/Source/AvatarCore_Manager/AvatarCore_Manager.Build.cs index e785ee8..d3e4414 100644 --- a/Unreal/Plugins/AvatarCore_Manager/Source/AvatarCore_Manager/AvatarCore_Manager.Build.cs +++ b/Unreal/Plugins/AvatarCore_Manager/Source/AvatarCore_Manager/AvatarCore_Manager.Build.cs @@ -42,6 +42,7 @@ public class AvatarCore_Manager : ModuleRules "AvatarCore_TTS", "AvatarCore_STT", "AvatarCore_AI", + "AvatarCore_Shared", "DeveloperSettings" // ... add private dependencies that you statically link with here ... } diff --git a/Unreal/Plugins/AvatarCore_Manager/Source/AvatarCore_Manager/Private/FL_AvatarCoreManager.cpp b/Unreal/Plugins/AvatarCore_Manager/Source/AvatarCore_Manager/Private/FL_AvatarCoreManager.cpp index f90c5ce..7242df3 100644 --- a/Unreal/Plugins/AvatarCore_Manager/Source/AvatarCore_Manager/Private/FL_AvatarCoreManager.cpp +++ b/Unreal/Plugins/AvatarCore_Manager/Source/AvatarCore_Manager/Private/FL_AvatarCoreManager.cpp @@ -17,144 +17,3 @@ void UFL_AvatarCoreManager::BindTTSToAIManager(UTTSManagerBase* TTSManager, UAIB AIManager->OnAudioChunk.AddDynamic(TTSManager, &UTTSManagerBase::AddAudioChunk); } } - -ETTSLanguage UFL_AvatarCoreManager::ConvertLanguageToTTS(ELanguage Language) -{ - switch (Language) - { - case ELanguage::ar: - return ETTSLanguage::ar; - - case ELanguage::de: - return ETTSLanguage::de; - - case ELanguage::en: - return ETTSLanguage::en; - - case ELanguage::fr: - return ETTSLanguage::fr; - - case ELanguage::el: - return ETTSLanguage::el; - - case ELanguage::hi: - return ETTSLanguage::hi; - - case ELanguage::it: - return ETTSLanguage::it; - - case ELanguage::ja: - return ETTSLanguage::ja; - - case ELanguage::ko: - return ETTSLanguage::ko; - - case ELanguage::zh: - return ETTSLanguage::zh; - - case ELanguage::nl: - return ETTSLanguage::nl; - - case ELanguage::pl: - return ETTSLanguage::pl; - - case ELanguage::pt: - return ETTSLanguage::pt; - - case ELanguage::ro: - return ETTSLanguage::ro; - - case ELanguage::ru: - return ETTSLanguage::ru; - - case ELanguage::es: - return ETTSLanguage::es; - - case ELanguage::cs: - return ETTSLanguage::cs; - - case ELanguage::tr: - return ETTSLanguage::tr; - - case ELanguage::uk: - return ETTSLanguage::uk; - - case ELanguage::hu: - return ETTSLanguage::hu; - - case ELanguage::NONE: - default: - return ETTSLanguage::NONE; - } -} - -ESTTLanguage UFL_AvatarCoreManager::ConvertLanguageToSTT(ELanguage Language) -{ - switch (Language) - { - case ELanguage::ar: - return ESTTLanguage::ar; - - case ELanguage::de: - return ESTTLanguage::de; - - case ELanguage::en: - return ESTTLanguage::en; - - case ELanguage::fr: - return ESTTLanguage::fr; - - case ELanguage::el: - return ESTTLanguage::el; - - case ELanguage::hi: - return ESTTLanguage::hi; - - case ELanguage::it: - return ESTTLanguage::it; - - case ELanguage::ja: - return ESTTLanguage::ja; - - case ELanguage::ko: - return ESTTLanguage::ko; - - case ELanguage::zh: - return ESTTLanguage::zh; - - case ELanguage::nl: - return ESTTLanguage::nl; - - case ELanguage::pl: - return ESTTLanguage::pl; - - case ELanguage::pt: - return ESTTLanguage::pt; - - case ELanguage::ro: - return ESTTLanguage::ro; - - case ELanguage::ru: - return ESTTLanguage::ru; - - case ELanguage::es: - return ESTTLanguage::es; - - case ELanguage::cs: - return ESTTLanguage::cs; - - case ELanguage::tr: - return ESTTLanguage::tr; - - case ELanguage::uk: - return ESTTLanguage::uk; - - case ELanguage::hu: - return ESTTLanguage::hu; - - case ELanguage::NONE: - default: - return ESTTLanguage::NONE; - - } -} diff --git a/Unreal/Plugins/AvatarCore_Manager/Source/AvatarCore_Manager/Public/AvatarCore_ManagerEnums.h b/Unreal/Plugins/AvatarCore_Manager/Source/AvatarCore_Manager/Public/AvatarCore_ManagerEnums.h index 10065ae..8a9a377 100644 --- a/Unreal/Plugins/AvatarCore_Manager/Source/AvatarCore_Manager/Public/AvatarCore_ManagerEnums.h +++ b/Unreal/Plugins/AvatarCore_Manager/Source/AvatarCore_Manager/Public/AvatarCore_ManagerEnums.h @@ -32,51 +32,4 @@ enum class EAvatarState : uint8 { Talking UMETA(DisplayName = "Avatar talking"), IdleActive UMETA(DisplayName = "Avatar idle active"), IdlePassive UMETA(DisplayName = "Avatar idle passive") -}; - -UENUM(BlueprintType) -enum class ELanguage : uint8 { - NONE UMETA(DisplayName = "Unset"), - en UMETA(DisplayName = "English"), - fr UMETA(DisplayName = "French"), - de UMETA(DisplayName = "German"), - es UMETA(DisplayName = "Spanish"), - pt UMETA(DisplayName = "Portuguese"), - zh UMETA(DisplayName = "Chinese"), - ja UMETA(DisplayName = "Japanese"), - hi UMETA(DisplayName = "Hindi"), - it UMETA(DisplayName = "Italian"), - ko UMETA(DisplayName = "Korean"), - nl UMETA(DisplayName = "Dutch"), - pl UMETA(DisplayName = "Polish"), - ru UMETA(DisplayName = "Russian"), - sv UMETA(DisplayName = "Swedish"), - tr UMETA(DisplayName = "Turkish"), - tl UMETA(DisplayName = "Filipino"), - bg UMETA(DisplayName = "Bulgarian"), - ro UMETA(DisplayName = "Romanian"), - ar UMETA(DisplayName = "Arabic"), - cs UMETA(DisplayName = "Czech"), - el UMETA(DisplayName = "Greek"), - fi UMETA(DisplayName = "Finnish"), - hr UMETA(DisplayName = "Croatian"), - ms UMETA(DisplayName = "Malay"), - sk UMETA(DisplayName = "Slovak"), - da UMETA(DisplayName = "Danish"), - ta UMETA(DisplayName = "Tamil"), - uk UMETA(DisplayName = "Ukrainian"), - hu UMETA(DisplayName = "Hungarian"), - no UMETA(DisplayName = "Norwegian"), - vi UMETA(DisplayName = "Vietnamese"), - bn UMETA(DisplayName = "Bengali"), - th UMETA(DisplayName = "Thai"), - he UMETA(DisplayName = "Hebrew"), - ka UMETA(DisplayName = "Georgian"), - id UMETA(DisplayName = "Indonesian"), - te UMETA(DisplayName = "Telugu"), - gu UMETA(DisplayName = "Gujarati"), - kn UMETA(DisplayName = "Kannada"), - ml UMETA(DisplayName = "Malayalam"), - mr UMETA(DisplayName = "Marathi"), - pa UMETA(DisplayName = "Punjabi"), }; \ No newline at end of file diff --git a/Unreal/Plugins/AvatarCore_Manager/Source/AvatarCore_Manager/Public/FL_AvatarCoreManager.h b/Unreal/Plugins/AvatarCore_Manager/Source/AvatarCore_Manager/Public/FL_AvatarCoreManager.h index b18b71d..ee2f0e8 100644 --- a/Unreal/Plugins/AvatarCore_Manager/Source/AvatarCore_Manager/Public/FL_AvatarCoreManager.h +++ b/Unreal/Plugins/AvatarCore_Manager/Source/AvatarCore_Manager/Public/FL_AvatarCoreManager.h @@ -48,10 +48,4 @@ class AVATARCORE_MANAGER_API UFL_AvatarCoreManager : public UBlueprintFunctionLi UFUNCTION(BlueprintCallable, Category = "AvatarCoreManager") static void BindTTSToAIManager(UTTSManagerBase* TTSManager, UAIBaseManager* AIManager); - - UFUNCTION(BlueprintCallable, Category = "AvatarCoreManager") - static ETTSLanguage ConvertLanguageToTTS(ELanguage Language); - - UFUNCTION(BlueprintCallable, Category = "AvatarCoreManager") - static ESTTLanguage ConvertLanguageToSTT(ELanguage Language); }; diff --git a/Unreal/Plugins/AvatarCore_STT/AvatarCore_STT.uplugin b/Unreal/Plugins/AvatarCore_STT/AvatarCore_STT.uplugin index 2a63c31..9fa281a 100644 --- a/Unreal/Plugins/AvatarCore_STT/AvatarCore_STT.uplugin +++ b/Unreal/Plugins/AvatarCore_STT/AvatarCore_STT.uplugin @@ -29,6 +29,10 @@ { "Name": "AudioCapture", "Enabled": true + }, + { + "Name": "AvatarCore_Shared", + "Enabled": true } ] } \ No newline at end of file diff --git a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/AvatarCore_STT.Build.cs b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/AvatarCore_STT.Build.cs index c71f1be..a4bd1ac 100644 --- a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/AvatarCore_STT.Build.cs +++ b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/AvatarCore_STT.Build.cs @@ -101,6 +101,7 @@ public class AvatarCore_STT : ModuleRules "Json", "JsonUtilities", "WebRTC", + "AvatarCore_Shared", // ... add private dependencies that you statically link with here ... } ); diff --git a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Processor/Azure/STTProcessorAzure.cpp b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Processor/Azure/STTProcessorAzure.cpp index 0453a83..1bd65d7 100644 --- a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Processor/Azure/STTProcessorAzure.cpp +++ b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Processor/Azure/STTProcessorAzure.cpp @@ -21,14 +21,14 @@ void USTTProcessorAzure::InitSTTProcessor(USTTManagerBase* BaseSTTManager, USTTB return; } - if (AzureProcessorConfig->AzureAPIKey.IsEmpty()) { + if (AzureProcessorConfig->AzureSettings.AzureAPIKey.IsEmpty()) { STTManager->OnSTTError.Broadcast(TEXT("Azure API Key not set. Needs to be done before initializing modules.")); return; } // Convert FString to std::string - std::string SubscriptionKey = TCHAR_TO_UTF8(*AzureProcessorConfig->AzureAPIKey); - std::string Region = TCHAR_TO_UTF8(*AzureProcessorConfig->AzureRegion); + std::string SubscriptionKey = TCHAR_TO_UTF8(*AzureProcessorConfig->AzureSettings.AzureAPIKey); + std::string Region = TCHAR_TO_UTF8(*AzureProcessorConfig->AzureSettings.AzureRegion); // Create the SpeechConfig object config = SpeechSDK::SpeechConfig::FromSubscription(SubscriptionKey, Region); @@ -89,7 +89,7 @@ void USTTProcessorAzure::OnChunkReceived(TArray PCMData, FAudioInformatio StopRecognition(false); } -void USTTProcessorAzure::ChangeAzureLanguage(TArray InLanguages) +void USTTProcessorAzure::ChangeAzureLanguage(TArray InLanguages) { AzureProcessorConfig->BaseSettings.STTLanguages = InLanguages; if (bDebugMode && IsValid(STTManager)) @@ -232,52 +232,52 @@ void USTTProcessorAzure::OnAzureError(FString Error) } } -FString USTTProcessorAzure::AzureEnumToString(ESTTLanguage Language) +FString USTTProcessorAzure::AzureEnumToString(ELanguage Language) { switch (Language) { - case ESTTLanguage::en: return "en-US"; - case ESTTLanguage::de: return "de-DE"; - case ESTTLanguage::fr: return "fr-FR"; - case ESTTLanguage::es: return "es-ES"; - case ESTTLanguage::pt: return "pt-BR"; - case ESTTLanguage::zh: return "zh-CN"; - case ESTTLanguage::ja: return "ja-JP"; - case ESTTLanguage::hi: return "hi-IN"; - case ESTTLanguage::it: return "it-IT"; - case ESTTLanguage::ko: return "ko-KR"; - case ESTTLanguage::nl: return "nl-NL"; - case ESTTLanguage::pl: return "pl-PL"; - case ESTTLanguage::ru: return "ru-RU"; - case ESTTLanguage::sv: return "sv-SE"; - case ESTTLanguage::tr: return "tr-TR"; - case ESTTLanguage::tl: return "fil-PH"; - case ESTTLanguage::bg: return "bg-BG"; - case ESTTLanguage::ro: return "ro-RO"; - case ESTTLanguage::ar: return "ar-SA"; - case ESTTLanguage::cs: return "cs-CZ"; - case ESTTLanguage::el: return "el-GR"; - case ESTTLanguage::fi: return "fi-FI"; - case ESTTLanguage::hr: return "hr-HR"; - case ESTTLanguage::ms: return "ms-MY"; - case ESTTLanguage::sk: return "sk-SK"; - case ESTTLanguage::da: return "da-DK"; - case ESTTLanguage::ta: return "ta-IN"; - case ESTTLanguage::uk: return "uk-UA"; - case ESTTLanguage::hu: return "hu-HU"; - case ESTTLanguage::no: return "nb-NO"; - case ESTTLanguage::vi: return "vi-VN"; - case ESTTLanguage::bn: return "bn-IN"; - case ESTTLanguage::th: return "th-TH"; - case ESTTLanguage::he: return "he-IL"; - case ESTTLanguage::ka: return "ka-GE"; - case ESTTLanguage::id: return "id-ID"; - case ESTTLanguage::te: return "te-IN"; - case ESTTLanguage::gu: return "gu-IN"; - case ESTTLanguage::kn: return "kn-IN"; - case ESTTLanguage::ml: return "ml-IN"; - case ESTTLanguage::mr: return "mr-IN"; - case ESTTLanguage::pa: return "pa-IN"; + case ELanguage::en: return "en-US"; + case ELanguage::de: return "de-DE"; + case ELanguage::fr: return "fr-FR"; + case ELanguage::es: return "es-ES"; + case ELanguage::pt: return "pt-BR"; + case ELanguage::zh: return "zh-CN"; + case ELanguage::ja: return "ja-JP"; + case ELanguage::hi: return "hi-IN"; + case ELanguage::it: return "it-IT"; + case ELanguage::ko: return "ko-KR"; + case ELanguage::nl: return "nl-NL"; + case ELanguage::pl: return "pl-PL"; + case ELanguage::ru: return "ru-RU"; + case ELanguage::sv: return "sv-SE"; + case ELanguage::tr: return "tr-TR"; + case ELanguage::tl: return "fil-PH"; + case ELanguage::bg: return "bg-BG"; + case ELanguage::ro: return "ro-RO"; + case ELanguage::ar: return "ar-SA"; + case ELanguage::cs: return "cs-CZ"; + case ELanguage::el: return "el-GR"; + case ELanguage::fi: return "fi-FI"; + case ELanguage::hr: return "hr-HR"; + case ELanguage::ms: return "ms-MY"; + case ELanguage::sk: return "sk-SK"; + case ELanguage::da: return "da-DK"; + case ELanguage::ta: return "ta-IN"; + case ELanguage::uk: return "uk-UA"; + case ELanguage::hu: return "hu-HU"; + case ELanguage::no: return "nb-NO"; + case ELanguage::vi: return "vi-VN"; + case ELanguage::bn: return "bn-IN"; + case ELanguage::th: return "th-TH"; + case ELanguage::he: return "he-IL"; + case ELanguage::ka: return "ka-GE"; + case ELanguage::id: return "id-ID"; + case ELanguage::te: return "te-IN"; + case ELanguage::gu: return "gu-IN"; + case ELanguage::kn: return "kn-IN"; + case ELanguage::ml: return "ml-IN"; + case ELanguage::mr: return "mr-IN"; + case ELanguage::pa: return "pa-IN"; default: return "UNDEFINED"; } } diff --git a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Processor/Parakeet/STTParakeetProcessorBase.cpp b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Processor/Parakeet/STTParakeetProcessorBase.cpp index 2efdcac..8f468dd 100644 --- a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Processor/Parakeet/STTParakeetProcessorBase.cpp +++ b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Processor/Parakeet/STTParakeetProcessorBase.cpp @@ -36,12 +36,12 @@ void USTTParakeetProcessorBase::InitSTTProcessor(USTTManagerBase* BaseSTTManager } #if WITH_EDITOR - if (ParakeetConfig->KeepAliveRule == ESTTKeepAliveRule::Never) + if (ParakeetConfig->ParakeetSettings.KeepAliveRule == EKeepAliveRule::Never) bKeepAlive = false; else bKeepAlive = true; #else - if (ParakeetConfig->KeepAliveRule == ESTTKeepAliveRule::Always) + if (ParakeetConfig->ParakeetSettings.KeepAliveRule == EKeepAliveRule::Always) bKeepAlive = true; else bKeepAlive = false; @@ -75,8 +75,8 @@ void USTTParakeetProcessorBase::InitSTTProcessor(USTTManagerBase* BaseSTTManager TSharedRef Addr = Subsystem->CreateInternetAddr(); bool bIsValid = false; - Addr->SetIp(*ParakeetConfig->Host, bIsValid); - Addr->SetPort(ParakeetConfig->Port > 0 ? ParakeetConfig->Port : ParakeetDefaultPort); + Addr->SetIp(*ParakeetConfig->ParakeetSettings.Host, bIsValid); + Addr->SetPort(ParakeetConfig->ParakeetSettings.Port > 0 ? ParakeetConfig->ParakeetSettings.Port : ParakeetDefaultPort); if (!bIsValid) { STTManager->OnSTTError.Broadcast(TEXT("Invalid address for Parakeet")); @@ -315,9 +315,9 @@ void USTTParakeetProcessorBase::SendConfiguration() TSharedPtr ConfigObj = MakeShared(); TSharedPtr ParamsObj = MakeShared(); - ParamsObj->SetStringField(TEXT("pretrained_model"), ParakeetConfig->PretrainedModel); - ParamsObj->SetStringField(TEXT("device"), ParakeetConfig->Device); - ParamsObj->SetNumberField(TEXT("update_interval"), ParakeetConfig->UpdateIntervalSec); + ParamsObj->SetStringField(TEXT("pretrained_model"), ParakeetConfig->ParakeetSettings.PretrainedModel); + ParamsObj->SetStringField(TEXT("device"), ParakeetConfig->ParakeetSettings.Device); + ParamsObj->SetNumberField(TEXT("update_interval"), ParakeetConfig->ParakeetSettings.UpdateIntervalSec); ConfigObj->SetStringField(TEXT("type"), TEXT("config")); ConfigObj->SetObjectField(TEXT("params"), ParamsObj); @@ -429,9 +429,9 @@ bool USTTParakeetProcessorBase::StartParakeetProcess() } const FString Args = FString::Printf(TEXT("\"%s\" serve-config --host %s --port %d"), - *ParakeetConfig->PythonPath, - *ParakeetConfig->Host, - ParakeetConfig->Port > 0 ? ParakeetConfig->Port : ParakeetDefaultPort); + *ParakeetConfig->ParakeetSettings.PythonPath, + *ParakeetConfig->ParakeetSettings.Host, + ParakeetConfig->ParakeetSettings.Port > 0 ? ParakeetConfig->ParakeetSettings.Port : ParakeetDefaultPort); uint32 ProcId = 0; diff --git a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Processor/Whisper/STTProcessorWhisper.cpp b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Processor/Whisper/STTProcessorWhisper.cpp index 9cd7eeb..e88023b 100644 --- a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Processor/Whisper/STTProcessorWhisper.cpp +++ b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Processor/Whisper/STTProcessorWhisper.cpp @@ -51,7 +51,7 @@ void USTTProcessorWhisper::InitSTTProcessor(USTTManagerBase* BaseSTTManager, UST return; } - if (WhisperProcessorConfig->OpenAI_API_Key.IsEmpty()) { + if (WhisperProcessorConfig->WhisperSettings.OpenAI_API_Key.IsEmpty()) { if (IsValid(STTManager)) STTManager->OnSTTError.Broadcast(TEXT("OpenAI API Key not set. Needs to be done before initializing modules.")); return; @@ -61,7 +61,7 @@ void USTTProcessorWhisper::InitSTTProcessor(USTTManagerBase* BaseSTTManager, UST PerformHealthCheck(); - STTManager->OnSTTLog.Broadcast(FString::Printf(TEXT("STTProcessor OpenAI %s initialized successfully."), *TranscribeModelEnumToString(WhisperProcessorConfig->Model))); + STTManager->OnSTTLog.Broadcast(FString::Printf(TEXT("STTProcessor OpenAI %s initialized successfully."), *TranscribeModelEnumToString(WhisperProcessorConfig->WhisperSettings.Model))); } void USTTProcessorWhisper::ClearSTTProcessor() @@ -114,7 +114,7 @@ void USTTProcessorWhisper::StartTranscriptionFromBuffer() { const float Frames = static_cast(PCMDataCopy.Num()) / static_cast(AudioInfoCopy.NumChannels); const float DurationSeconds = Frames / static_cast(AudioInfoCopy.SampleRate); - if (DurationSeconds < WhisperProcessorConfig->MinDuration) + if (DurationSeconds < WhisperProcessorConfig->WhisperSettings.MinDuration) { return; } @@ -198,9 +198,9 @@ void USTTProcessorWhisper::BuildMultipartBody(const TArray& WavData, cons FString BoundaryLine = FString::Printf(TEXT("--%s\r\n"), *Boundary); AppendStringToBody(OutBody, BoundaryLine); AppendStringToBody(OutBody, TEXT("Content-Disposition: form-data; name=\"model\"\r\n\r\n")); - AppendStringToBody(OutBody, TranscribeModelEnumToString(WhisperProcessorConfig->Model) + TEXT("\r\n")); + AppendStringToBody(OutBody, TranscribeModelEnumToString(WhisperProcessorConfig->WhisperSettings.Model) + TEXT("\r\n")); - if (WhisperProcessorConfig->Model == EOpenAITranscriptionModel::Whisper1) + if (WhisperProcessorConfig->WhisperSettings.Model == EOpenAITranscriptionModel::Whisper1) { BoundaryLine = FString::Printf(TEXT("--%s\r\n"), *Boundary); AppendStringToBody(OutBody, BoundaryLine); @@ -233,7 +233,7 @@ void USTTProcessorWhisper::NormalizeWhisperURL() if (!WhisperProcessorConfig) return; - NormalizedWhisperURL = WhisperProcessorConfig->WhisperURL; + NormalizedWhisperURL = WhisperProcessorConfig->WhisperSettings.WhisperURL; if (!NormalizedWhisperURL.StartsWith(TEXT("http://")) && !NormalizedWhisperURL.StartsWith(TEXT("https://"))) { NormalizedWhisperURL = FString::Printf(TEXT("https://%s"), *NormalizedWhisperURL); @@ -252,7 +252,7 @@ void USTTProcessorWhisper::PerformHealthCheck() TSharedRef Request = HttpModule.CreateRequest(); Request->SetURL(NormalizedWhisperURL); Request->SetVerb(TEXT("GET")); - FString AuthHeader = FString::Printf(TEXT("Bearer %s"), *WhisperProcessorConfig->OpenAI_API_Key); + FString AuthHeader = FString::Printf(TEXT("Bearer %s"), *WhisperProcessorConfig->WhisperSettings.OpenAI_API_Key); Request->SetHeader(TEXT("Authorization"), AuthHeader); Request->OnProcessRequestComplete().BindLambda([ WeakManager = TWeakObjectPtr(STTManager)](FHttpRequestPtr Req, FHttpResponsePtr Resp, bool bWasSuccessful) @@ -292,7 +292,7 @@ void USTTProcessorWhisper::SendWhisperRequest(TArray&& WavData) Request->SetURL(NormalizedWhisperURL); Request->SetVerb(TEXT("POST")); - FString AuthHeader = FString::Printf(TEXT("Bearer %s"), *WhisperProcessorConfig->OpenAI_API_Key); + FString AuthHeader = FString::Printf(TEXT("Bearer %s"), *WhisperProcessorConfig->WhisperSettings.OpenAI_API_Key); Request->SetHeader(TEXT("Authorization"), AuthHeader); Request->SetHeader(TEXT("Accept"), TEXT("application/json")); diff --git a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/STTManagerBase.cpp b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/STTManagerBase.cpp index 384dd3a..61fa375 100644 --- a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/STTManagerBase.cpp +++ b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/STTManagerBase.cpp @@ -175,7 +175,7 @@ bool USTTManagerBase::IsBlocked() return (CurrentSpeechState==ESTTTalkingState::BLOCKED); } -void USTTManagerBase::SetLanguage(TArray NewLanguages) +void USTTManagerBase::SetSTTLanguage(TArray NewLanguages) { if (!IsValid(ProcessorConfig)) return; diff --git a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Processor/Azure/STTAzureProcessorConfig.h b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Processor/Azure/STTAzureProcessorConfig.h index 9a5aa61..9c54390 100644 --- a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Processor/Azure/STTAzureProcessorConfig.h +++ b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Processor/Azure/STTAzureProcessorConfig.h @@ -6,6 +6,18 @@ #include "Processor/STTBaseProcessorConfig.h" #include "STTAzureProcessorConfig.generated.h" +USTRUCT(BlueprintType) +struct FSTTAzureSettings +{ + GENERATED_BODY() + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreSTT|Azure", meta = (ExposeOnSpawn = "true")) + FString AzureAPIKey = ""; + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreSTT|Azure", meta = (ExposeOnSpawn = "true")) + FString AzureRegion = "germanywestcentral"; + +}; + /** * */ @@ -19,7 +31,5 @@ public: USTTAzureProcessorConfig(const FObjectInitializer& ObjectInitializer); UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreSTT|Azure", meta = (ExposeOnSpawn = "true")) - FString AzureAPIKey = ""; - UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreSTT|Azure", meta = (ExposeOnSpawn = "true")) - FString AzureRegion = "germanywestcentral"; + FSTTAzureSettings AzureSettings; }; diff --git a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Processor/Azure/STTProcessorAzure.h b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Processor/Azure/STTProcessorAzure.h index 2b7c806..3bfeb9d 100644 --- a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Processor/Azure/STTProcessorAzure.h +++ b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Processor/Azure/STTProcessorAzure.h @@ -32,7 +32,7 @@ class AVATARCORE_STT_API USTTProcessorAzure : public USTTProcessorBase public: UFUNCTION(BlueprintCallable, Category = STTManager) - virtual void ChangeAzureLanguage(TArray InLanguages); + virtual void ChangeAzureLanguage(TArray InLanguages); private: @@ -62,7 +62,7 @@ public: void OnAzureError(FString Error); UFUNCTION(BlueprintPure, Category = STTManager) - FString AzureEnumToString(ESTTLanguage Language); + FString AzureEnumToString(ELanguage Language); USTTAzureProcessorConfig* AzureProcessorConfig; }; diff --git a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Processor/Parakeet/STTParakeetProcessorConfig.h b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Processor/Parakeet/STTParakeetProcessorConfig.h index 7dd6594..50cf4a6 100644 --- a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Processor/Parakeet/STTParakeetProcessorConfig.h +++ b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Processor/Parakeet/STTParakeetProcessorConfig.h @@ -6,18 +6,11 @@ #include "Processor/STTBaseProcessorConfig.h" #include "STTParakeetProcessorConfig.generated.h" -/** - * Configuration for the Parakeet STT Processor (NVIDIA NeMo ASR). - */ -UCLASS(Blueprintable, BlueprintType) -class AVATARCORE_STT_API USTTParakeetProcessorConfig : public USTTBaseProcessorConfig +USTRUCT(BlueprintType) +struct FSTTParakeetSettings { GENERATED_BODY() -public: - - USTTParakeetProcessorConfig(const FObjectInitializer& ObjectInitializer); - UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreSTT|Parakeet", meta = (ExposeOnSpawn = "true")) FString PythonPath = "python"; @@ -31,7 +24,7 @@ public: int32 Port = 40200; UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreSTT|Parakeet", meta = (ExposeOnSpawn = "true")) - ESTTKeepAliveRule KeepAliveRule = ESTTKeepAliveRule::EditorOnly; + EKeepAliveRule KeepAliveRule = EKeepAliveRule::EditorOnly; UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreSTT|Parakeet", meta = (ExposeOnSpawn = "true")) FString Device = "cuda:0"; @@ -39,4 +32,21 @@ public: // How often (seconds) the Python server produces intermediate transcription updates UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreSTT|Parakeet", meta = (ExposeOnSpawn = "true", ClampMin = "0.1", ClampMax = "5.0")) float UpdateIntervalSec = 0.5f; + +}; + +/** + * Configuration for the Parakeet STT Processor (NVIDIA NeMo ASR). + */ +UCLASS(Blueprintable, BlueprintType) +class AVATARCORE_STT_API USTTParakeetProcessorConfig : public USTTBaseProcessorConfig +{ + GENERATED_BODY() + +public: + + USTTParakeetProcessorConfig(const FObjectInitializer& ObjectInitializer); + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreSTT|Parakeet", meta = (ExposeOnSpawn = "true", ClampMin = "0.1", ClampMax = "5.0")) + FSTTParakeetSettings ParakeetSettings; }; diff --git a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Processor/STTBaseProcessorConfig.h b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Processor/STTBaseProcessorConfig.h index 4b32940..4a2414d 100644 --- a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Processor/STTBaseProcessorConfig.h +++ b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Processor/STTBaseProcessorConfig.h @@ -7,13 +7,6 @@ #include "STTStructs.h" #include "STTBaseProcessorConfig.generated.h" -UENUM(BlueprintType) -enum class ESTTKeepAliveRule : uint8 { - EditorOnly UMETA(DisplayName = "Only keep STT Module open in Editor Mode"), - Never UMETA(DisplayName = "Never keep STT Module alive."), - Always UMETA(DisplayName = "Keep STT Module alive in Editor and Shipping mode.") -}; - class USTTProcessorBase; /** diff --git a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Processor/Whisper/STTWhisperProcessorConfig.h b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Processor/Whisper/STTWhisperProcessorConfig.h index dcd6948..dc608ba 100644 --- a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Processor/Whisper/STTWhisperProcessorConfig.h +++ b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Processor/Whisper/STTWhisperProcessorConfig.h @@ -14,6 +14,23 @@ enum class EOpenAITranscriptionModel : uint8 Transcribe4o UMETA(DisplayName = "4o Transcribe") }; +USTRUCT(BlueprintType) +struct FSTTWhisperSettings +{ + GENERATED_BODY() + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreSTT|Whisper", meta = (ExposeOnSpawn = "true")) + FString OpenAI_API_Key = ""; + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreSTT|Whisper", meta = (ExposeOnSpawn = "true")) + FString WhisperURL = "api.openai.com/v1/audio/transcriptions"; + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreSTT|Whisper", meta = (ExposeOnSpawn = "true")) + EOpenAITranscriptionModel Model = EOpenAITranscriptionModel::Transcribe4o; + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreSTT|Whisper", meta = (ExposeOnSpawn = "true")) + float MinDuration = 0.75f; + +}; + + /** * */ @@ -27,12 +44,6 @@ public: USTTWhisperProcessorConfig(const FObjectInitializer& ObjectInitializer); UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreSTT|Whisper", meta = (ExposeOnSpawn = "true")) - FString OpenAI_API_Key = ""; - UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreSTT|Whisper", meta = (ExposeOnSpawn = "true")) - FString WhisperURL = "api.openai.com/v1/audio/transcriptions"; - UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreSTT|Whisper", meta = (ExposeOnSpawn = "true")) - EOpenAITranscriptionModel Model = EOpenAITranscriptionModel::Transcribe4o; - UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreSTT|Whisper", meta = (ExposeOnSpawn = "true")) - float MinDuration = 0.75f; + FSTTWhisperSettings WhisperSettings; }; diff --git a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/STTManagerBase.h b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/STTManagerBase.h index 99a2772..41744cd 100644 --- a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/STTManagerBase.h +++ b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/STTManagerBase.h @@ -99,7 +99,7 @@ public: bool IsBlocked(); UFUNCTION(BlueprintCallable, Category = "AvatarCoreSTT") - void SetLanguage(TArray NewLanguages); + void SetSTTLanguage(TArray NewLanguages); UFUNCTION(BlueprintCallable, Category = "AvatarCoreSTT") void AddSpecialWord(FString NewWord); diff --git a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/STTStructs.h b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/STTStructs.h index f7cc50e..157bb89 100644 --- a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/STTStructs.h +++ b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/STTStructs.h @@ -1,6 +1,7 @@ #pragma once #include "CoreMinimal.h" +#include "AvatarCoreSharedEnums.h" #include "STTStructs.generated.h" USTRUCT(BlueprintType) @@ -70,54 +71,6 @@ enum class ESpeexDSPState : uint8 SPEEXPREPROCESS_SET_AGC_TARGET = 46 UMETA(DisplayName = "preprocessor Automatic Gain Control level (int32)") }; -UENUM(BlueprintType) -enum class ESTTLanguage : uint8 -{ - NONE UMETA(DisplayName = "Unset"), - en UMETA(DisplayName = "English"), - fr UMETA(DisplayName = "French"), - de UMETA(DisplayName = "German"), - es UMETA(DisplayName = "Spanish"), - pt UMETA(DisplayName = "Portuguese"), - zh UMETA(DisplayName = "Chinese"), - ja UMETA(DisplayName = "Japanese"), - hi UMETA(DisplayName = "Hindi"), - it UMETA(DisplayName = "Italian"), - ko UMETA(DisplayName = "Korean"), - nl UMETA(DisplayName = "Dutch"), - pl UMETA(DisplayName = "Polish"), - ru UMETA(DisplayName = "Russian"), - sv UMETA(DisplayName = "Swedish"), - tr UMETA(DisplayName = "Turkish"), - tl UMETA(DisplayName = "Filipino"), - bg UMETA(DisplayName = "Bulgarian"), - ro UMETA(DisplayName = "Romanian"), - ar UMETA(DisplayName = "Arabic"), - cs UMETA(DisplayName = "Czech"), - el UMETA(DisplayName = "Greek"), - fi UMETA(DisplayName = "Finnish"), - hr UMETA(DisplayName = "Croatian"), - ms UMETA(DisplayName = "Malay"), - sk UMETA(DisplayName = "Slovak"), - da UMETA(DisplayName = "Danish"), - ta UMETA(DisplayName = "Tamil"), - uk UMETA(DisplayName = "Ukrainian"), - hu UMETA(DisplayName = "Hungarian"), - no UMETA(DisplayName = "Norwegian"), - vi UMETA(DisplayName = "Vietnamese"), - bn UMETA(DisplayName = "Bengali"), - th UMETA(DisplayName = "Thai"), - he UMETA(DisplayName = "Hebrew"), - ka UMETA(DisplayName = "Georgian"), - id UMETA(DisplayName = "Indonesian"), - te UMETA(DisplayName = "Telugu"), - gu UMETA(DisplayName = "Gujarati"), - kn UMETA(DisplayName = "Kannada"), - ml UMETA(DisplayName = "Malayalam"), - mr UMETA(DisplayName = "Marathi"), - pa UMETA(DisplayName = "Punjabi"), -}; - USTRUCT(BlueprintType) struct FWebRTCSettings { @@ -262,7 +215,7 @@ struct FSTTBaseSettings UPROPERTY(EditAnywhere, BlueprintReadWrite, meta = (ToolTip = "Settings of the SpeexDSP Module", Category = "STT|Base")) FSpeexDSPSettings SpeexDSPSettings; UPROPERTY(EditAnywhere, BlueprintReadWrite, meta = (ToolTip = "All languages the STT module should understand simultaneously.", Category = "STT|Base")) - TArray STTLanguages = { ESTTLanguage::de, ESTTLanguage::en }; + TArray STTLanguages = { ELanguage::de, ELanguage::en }; UPROPERTY(EditAnywhere, BlueprintReadWrite, meta = (ToolTip = "Transcriptions to always change to another word.", Category = "STT|Base")) TArray STTReplacements; UPROPERTY(EditAnywhere, BlueprintReadWrite, meta = (ToolTip = "Special words that the transcription service needs to know (e.g. b.ReX or Bruce-B).", Category = "STT|Base")) diff --git a/Unreal/Plugins/AvatarCore_Shared/AvatarCore_Shared.uplugin b/Unreal/Plugins/AvatarCore_Shared/AvatarCore_Shared.uplugin new file mode 100644 index 0000000..0b449dd --- /dev/null +++ b/Unreal/Plugins/AvatarCore_Shared/AvatarCore_Shared.uplugin @@ -0,0 +1,24 @@ +{ + "FileVersion": 3, + "Version": 1, + "VersionName": "1.0", + "FriendlyName": "AvatarCore_Shared", + "Description": "Shared assets, structs and enums. In theory we want to be AI, TTS and STT to be separate.", + "Category": "Other", + "CreatedBy": "b.ReX GmbH", + "CreatedByURL": "", + "DocsURL": "", + "MarketplaceURL": "", + "SupportURL": "", + "CanContainContent": true, + "IsBetaVersion": false, + "IsExperimentalVersion": false, + "Installed": false, + "Modules": [ + { + "Name": "AvatarCore_Shared", + "Type": "Runtime", + "LoadingPhase": "Default" + } + ] +} \ No newline at end of file diff --git a/Unreal/Plugins/AvatarCore_Shared/Resources/Icon128.png b/Unreal/Plugins/AvatarCore_Shared/Resources/Icon128.png new file mode 100644 index 0000000..caaef9b --- /dev/null +++ b/Unreal/Plugins/AvatarCore_Shared/Resources/Icon128.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:27a3a6ccd0cbc081eed7b1f0c6bfc2dee1bcdec2b7040725beab4ca6728b7c00 +size 6093 diff --git a/Unreal/Plugins/AvatarCore_Shared/Source/AvatarCore_Shared/AvatarCore_Shared.Build.cs b/Unreal/Plugins/AvatarCore_Shared/Source/AvatarCore_Shared/AvatarCore_Shared.Build.cs new file mode 100644 index 0000000..22af523 --- /dev/null +++ b/Unreal/Plugins/AvatarCore_Shared/Source/AvatarCore_Shared/AvatarCore_Shared.Build.cs @@ -0,0 +1,53 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +using UnrealBuildTool; + +public class AvatarCore_Shared : ModuleRules +{ + public AvatarCore_Shared(ReadOnlyTargetRules Target) : base(Target) + { + PCHUsage = ModuleRules.PCHUsageMode.UseExplicitOrSharedPCHs; + + PublicIncludePaths.AddRange( + new string[] { + // ... add public include paths required here ... + } + ); + + + PrivateIncludePaths.AddRange( + new string[] { + // ... add other private include paths required here ... + } + ); + + + PublicDependencyModuleNames.AddRange( + new string[] + { + "Core", + // ... add other public dependencies that you statically link with here ... + } + ); + + + PrivateDependencyModuleNames.AddRange( + new string[] + { + "CoreUObject", + "Engine", + "Slate", + "SlateCore", + // ... add private dependencies that you statically link with here ... + } + ); + + + DynamicallyLoadedModuleNames.AddRange( + new string[] + { + // ... add any modules that your module loads dynamically here ... + } + ); + } +} diff --git a/Unreal/Plugins/AvatarCore_Shared/Source/AvatarCore_Shared/Private/AvatarCore_Shared.cpp b/Unreal/Plugins/AvatarCore_Shared/Source/AvatarCore_Shared/Private/AvatarCore_Shared.cpp new file mode 100644 index 0000000..347afbc --- /dev/null +++ b/Unreal/Plugins/AvatarCore_Shared/Source/AvatarCore_Shared/Private/AvatarCore_Shared.cpp @@ -0,0 +1,20 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "AvatarCore_Shared.h" + +#define LOCTEXT_NAMESPACE "FAvatarCore_SharedModule" + +void FAvatarCore_SharedModule::StartupModule() +{ + // This code will execute after your module is loaded into memory; the exact timing is specified in the .uplugin file per-module +} + +void FAvatarCore_SharedModule::ShutdownModule() +{ + // This function may be called during shutdown to clean up your module. For modules that support dynamic reloading, + // we call this function before unloading the module. +} + +#undef LOCTEXT_NAMESPACE + +IMPLEMENT_MODULE(FAvatarCore_SharedModule, AvatarCore_Shared) \ No newline at end of file diff --git a/Unreal/Plugins/AvatarCore_Shared/Source/AvatarCore_Shared/Private/FL_AvatarCoreShared.cpp b/Unreal/Plugins/AvatarCore_Shared/Source/AvatarCore_Shared/Private/FL_AvatarCoreShared.cpp new file mode 100644 index 0000000..cceeea8 --- /dev/null +++ b/Unreal/Plugins/AvatarCore_Shared/Source/AvatarCore_Shared/Private/FL_AvatarCoreShared.cpp @@ -0,0 +1,27 @@ +// Fill out your copyright notice in the Description page of Project Settings. + + +#include "FL_AvatarCoreShared.h" + +ELanguage UFL_AvatarCoreShared::StringToLanguage(const FString& InString) +{ + const UEnum* EnumPtr = StaticEnum(); + + if (!EnumPtr) + { + return ELanguage::NONE; + } + + FString Normalized = InString.ToLower(); + + int64 Value = EnumPtr->GetValueByNameString(Normalized); + + return Value != INDEX_NONE + ? static_cast(Value) + : ELanguage::NONE; +} + +FString UFL_AvatarCoreShared::LanguageToString(ELanguage Language) +{ + return StaticEnum()->GetNameStringByValue((int64)Language); +} \ No newline at end of file diff --git a/Unreal/Plugins/AvatarCore_Shared/Source/AvatarCore_Shared/Public/AvatarCoreSharedEnums.h b/Unreal/Plugins/AvatarCore_Shared/Source/AvatarCore_Shared/Public/AvatarCoreSharedEnums.h new file mode 100644 index 0000000..7d9ae51 --- /dev/null +++ b/Unreal/Plugins/AvatarCore_Shared/Source/AvatarCore_Shared/Public/AvatarCoreSharedEnums.h @@ -0,0 +1,62 @@ +// Fill out your copyright notice in the Description page of Project Settings. + +#pragma once + +#include "CoreMinimal.h" +#include "UObject/NoExportTypes.h" +#include "AvatarCoreSharedEnums.generated.h" + +UENUM(BlueprintType) +enum class ELanguage : uint8 { + NONE UMETA(DisplayName = "Unset"), + en UMETA(DisplayName = "English"), + fr UMETA(DisplayName = "French"), + de UMETA(DisplayName = "German"), + es UMETA(DisplayName = "Spanish"), + pt UMETA(DisplayName = "Portuguese"), + zh UMETA(DisplayName = "Chinese"), + ja UMETA(DisplayName = "Japanese"), + hi UMETA(DisplayName = "Hindi"), + it UMETA(DisplayName = "Italian"), + ko UMETA(DisplayName = "Korean"), + nl UMETA(DisplayName = "Dutch"), + pl UMETA(DisplayName = "Polish"), + ru UMETA(DisplayName = "Russian"), + sv UMETA(DisplayName = "Swedish"), + tr UMETA(DisplayName = "Turkish"), + tl UMETA(DisplayName = "Filipino"), + bg UMETA(DisplayName = "Bulgarian"), + ro UMETA(DisplayName = "Romanian"), + ar UMETA(DisplayName = "Arabic"), + cs UMETA(DisplayName = "Czech"), + el UMETA(DisplayName = "Greek"), + fi UMETA(DisplayName = "Finnish"), + hr UMETA(DisplayName = "Croatian"), + ms UMETA(DisplayName = "Malay"), + sk UMETA(DisplayName = "Slovak"), + da UMETA(DisplayName = "Danish"), + ta UMETA(DisplayName = "Tamil"), + uk UMETA(DisplayName = "Ukrainian"), + hu UMETA(DisplayName = "Hungarian"), + no UMETA(DisplayName = "Norwegian"), + vi UMETA(DisplayName = "Vietnamese"), + bn UMETA(DisplayName = "Bengali"), + th UMETA(DisplayName = "Thai"), + he UMETA(DisplayName = "Hebrew"), + ka UMETA(DisplayName = "Georgian"), + id UMETA(DisplayName = "Indonesian"), + te UMETA(DisplayName = "Telugu"), + gu UMETA(DisplayName = "Gujarati"), + kn UMETA(DisplayName = "Kannada"), + ml UMETA(DisplayName = "Malayalam"), + mr UMETA(DisplayName = "Marathi"), + pa UMETA(DisplayName = "Punjabi"), +}; + +UENUM(BlueprintType) +enum class EKeepAliveRule : uint8 { + EditorOnly UMETA(DisplayName = "Only keep Module open in Editor Mode"), + Never UMETA(DisplayName = "Never keep Module alive."), + Always UMETA(DisplayName = "Keep Module alive in Editor and Shipping mode.") +}; + diff --git a/Unreal/Plugins/AvatarCore_Shared/Source/AvatarCore_Shared/Public/AvatarCore_Shared.h b/Unreal/Plugins/AvatarCore_Shared/Source/AvatarCore_Shared/Public/AvatarCore_Shared.h new file mode 100644 index 0000000..f37777b --- /dev/null +++ b/Unreal/Plugins/AvatarCore_Shared/Source/AvatarCore_Shared/Public/AvatarCore_Shared.h @@ -0,0 +1,14 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "Modules/ModuleManager.h" + +class FAvatarCore_SharedModule : public IModuleInterface +{ +public: + + /** IModuleInterface implementation */ + virtual void StartupModule() override; + virtual void ShutdownModule() override; +}; diff --git a/Unreal/Plugins/AvatarCore_Shared/Source/AvatarCore_Shared/Public/FL_AvatarCoreShared.h b/Unreal/Plugins/AvatarCore_Shared/Source/AvatarCore_Shared/Public/FL_AvatarCoreShared.h new file mode 100644 index 0000000..78917d8 --- /dev/null +++ b/Unreal/Plugins/AvatarCore_Shared/Source/AvatarCore_Shared/Public/FL_AvatarCoreShared.h @@ -0,0 +1,27 @@ +// Fill out your copyright notice in the Description page of Project Settings. + +#pragma once + +#include "CoreMinimal.h" +#include "Kismet/BlueprintFunctionLibrary.h" +#include "AvatarCoreSharedEnums.h" +#include "FL_AvatarCoreShared.generated.h" + +/** + * + */ +UCLASS() +class AVATARCORE_SHARED_API UFL_AvatarCoreShared : public UBlueprintFunctionLibrary +{ + GENERATED_BODY() + +public: + // Returns the ISO 639-1 code string for the given language enum (e.g. ELanguage::de "de"). + // Derived from the enum value name via UE reflection, so it stays in sync automatically. + UFUNCTION(BlueprintCallable, Category = "AvatarCoreShared") + static FString LanguageToString(ELanguage Language); + + // Returns the ELanguage for the given ISO 639-1 code string + UFUNCTION(BlueprintCallable, Category = "AvatarCoreShared") + static ELanguage StringToLanguage(const FString& InString); +}; diff --git a/Unreal/Plugins/AvatarCore_TTS/AvatarCore_TTS.uplugin b/Unreal/Plugins/AvatarCore_TTS/AvatarCore_TTS.uplugin index 3fd6004..4de11cd 100644 --- a/Unreal/Plugins/AvatarCore_TTS/AvatarCore_TTS.uplugin +++ b/Unreal/Plugins/AvatarCore_TTS/AvatarCore_TTS.uplugin @@ -25,6 +25,10 @@ { "Name": "BTools", "Enabled": true + }, + { + "Name": "AvatarCore_Shared", + "Enabled": true } ] } \ No newline at end of file diff --git a/Unreal/Plugins/AvatarCore_TTS/Source/.claude/settings.local.json b/Unreal/Plugins/AvatarCore_TTS/Source/.claude/settings.local.json new file mode 100644 index 0000000..945105a --- /dev/null +++ b/Unreal/Plugins/AvatarCore_TTS/Source/.claude/settings.local.json @@ -0,0 +1,9 @@ +{ + "permissions": { + "allow": [ + "Bash(Get-ChildItem -Path \"c:\\\\Gitprojects\\\\SchaniConcierge\\\\Unreal\\\\Plugins\\\\AvatarCore_TTS\\\\Source\" -Recurse -Filter \"CartesiaTTSManager.h\")", + "Bash(Select-Object -ExpandProperty FullName)", + "Bash(find c:\\\\\\\\Gitprojects\\\\\\\\SchaniConcierge\\\\\\\\Unreal\\\\\\\\Plugins\\\\\\\\AvatarCore_TTS\\\\\\\\Source -name \"CartesiaTTSManager.h\")" + ] + } +} diff --git a/Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/AvatarCore_TTS.Build.cs b/Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/AvatarCore_TTS.Build.cs index b429d5f..74e4aae 100644 --- a/Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/AvatarCore_TTS.Build.cs +++ b/Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/AvatarCore_TTS.Build.cs @@ -35,6 +35,7 @@ public class AvatarCore_TTS : ModuleRules "AudioMixer", "AudioExtensions", "WebRTC", + "AvatarCore_Shared", // ... add other public dependencies that you statically link with here ... } ); diff --git a/Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Private/Cartesia/CartesiaTTSManager.cpp b/Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Private/Cartesia/CartesiaTTSManager.cpp index cd3f2cd..95e8ada 100644 --- a/Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Private/Cartesia/CartesiaTTSManager.cpp +++ b/Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Private/Cartesia/CartesiaTTSManager.cpp @@ -1,8 +1,8 @@ -// Fill out your copyright notice in the Description page of Project Settings. +// Fill out your copyright notice in the Description page of Project Settings. #include "Cartesia/CartesiaTTSManager.h" - +#include "FL_AvatarCoreShared.h" #include "WebSocketsModule.h" #include "IWebSocket.h" #include "Async/Async.h" @@ -168,7 +168,7 @@ void UCartesiaTTSManager::SendTranscriptMessage(int32 TaskID, const FString& Tra TSharedPtr Obj = MakeShared(); Obj->SetStringField(TEXT("model_id"), CartesiaTTSConfig->CartesiaTTSSettings.CartesiaModelId); Obj->SetStringField(TEXT("transcript"), Transcript); - Obj->SetStringField(TEXT("language"), TTSLanguageToString(CartesiaTTSConfig->GlobalTTSSettings.Language)); + Obj->SetStringField(TEXT("language"), UFL_AvatarCoreShared::LanguageToString(CartesiaTTSConfig->GlobalTTSSettings.Language)); Obj->SetStringField(TEXT("context_id"), GetOrCreateContextId(TaskID)); Obj->SetBoolField(TEXT("continue"), bContinue); @@ -183,8 +183,8 @@ void UCartesiaTTSManager::SendTranscriptMessage(int32 TaskID, const FString& Tra OutputObj->SetNumberField(TEXT("sample_rate"), (TTSConfig ? TTSConfig->GlobalTTSSettings.AudioSampleRate : 24000)); Obj->SetObjectField(TEXT("output_format"), OutputObj); - Obj->SetBoolField(TEXT("add_timestamps"), false); - + Obj->SetBoolField(TEXT("add_timestamps"), true); + SendJsonForTask(TaskID, Obj); { @@ -217,7 +217,7 @@ void UCartesiaTTSManager::SendFlushMessage(int32 TaskID) TSharedPtr Obj = MakeShared(); Obj->SetStringField(TEXT("model_id"), CartesiaTTSConfig->CartesiaTTSSettings.CartesiaModelId); Obj->SetStringField(TEXT("transcript"), TEXT("")); - Obj->SetStringField(TEXT("language"), TTSLanguageToString(CartesiaTTSConfig->GlobalTTSSettings.Language)); + Obj->SetStringField(TEXT("language"), UFL_AvatarCoreShared::LanguageToString(CartesiaTTSConfig->GlobalTTSSettings.Language)); Obj->SetStringField(TEXT("context_id"), GetOrCreateContextId(TaskID)); Obj->SetBoolField(TEXT("continue"), true); Obj->SetBoolField(TEXT("flush"), true); @@ -233,7 +233,7 @@ void UCartesiaTTSManager::SendFlushMessage(int32 TaskID) OutputObj->SetNumberField(TEXT("sample_rate"), (TTSConfig ? TTSConfig->GlobalTTSSettings.AudioSampleRate : 24000)); Obj->SetObjectField(TEXT("output_format"), OutputObj); - Obj->SetBoolField(TEXT("add_timestamps"), false); + Obj->SetBoolField(TEXT("add_timestamps"), true); SendJsonForTask(TaskID, Obj); } @@ -292,8 +292,8 @@ void UCartesiaTTSManager::StartStreamingGeneration(int32 TaskID, const FString& Self->TTSLog(FString::Printf(TEXT("[Cartesia] WS connected for task %d"), TaskID)); }); - WS->OnMessage().AddLambda([WeakThis, TaskID](const FString& Message) - { + WS->OnMessage().AddLambda([WeakThis, TaskID](const FString &Message) + { if (!WeakThis.IsValid()) return; UCartesiaTTSManager* Self = WeakThis.Get(); //Self->TTSLog(FString::Printf(TEXT("[Cartesia][%d] <- %s"), TaskID, *Message)); @@ -330,6 +330,53 @@ void UCartesiaTTSManager::StartStreamingGeneration(int32 TaskID, const FString& return; } + if (Type.Equals(TEXT("timestamps"), ESearchCase::IgnoreCase)) + { + const TSharedPtr* WTSObj; + if (Obj->TryGetObjectField(TEXT("word_timestamps"), WTSObj)) + { + const TArray>* WordArr; + const TArray>* StartArr; + const TArray>* EndArr; + if ((*WTSObj)->TryGetArrayField(TEXT("words"), WordArr) && + (*WTSObj)->TryGetArrayField(TEXT("start"), StartArr) && + (*WTSObj)->TryGetArrayField(TEXT("end"), EndArr)) + { + const int32 Count = FMath::Min3(WordArr->Num(), StartArr->Num(), EndArr->Num()); + TArray Timestamps; + Timestamps.Reserve(Count); + for (int32 i = 0; i < Count; ++i) + { + FWordTimestamp WTS; + WTS.Word = (*WordArr)[i]->AsString(); + WTS.Start = (float)(*StartArr)[i]->AsNumber(); + WTS.End = (float)(*EndArr)[i]->AsNumber(); + Timestamps.Add(WTS); + } + AsyncTask(ENamedThreads::GameThread, [Self, TaskID, TS = MoveTemp(Timestamps)]() mutable + { + FScopeLock Lock(&Self->TaskQueueCriticalSection); + for (FTTSTask& Task : Self->ActiveTasks) + { + if (Task.TaskID == TaskID) + { + // Deduplicate: only add words whose start time is after the last stored end time. + // Cartesia sends one timestamps message per audio chunk and may repeat words. + const float LastEnd = Task.WordTimestamps.IsEmpty() ? 0.f : Task.WordTimestamps.Last().End; + for (const FWordTimestamp& WTS : TS) + { + if (WTS.Start >= LastEnd - 0.01f) + Task.WordTimestamps.Add(WTS); + } + break; + } + } + }); + } + } + return; + } + if (Type.Equals(TEXT("done"), ESearchCase::IgnoreCase)) { bool bDone = false; @@ -353,8 +400,7 @@ void UCartesiaTTSManager::StartStreamingGeneration(int32 TaskID, const FString& Self->TTSError(FString::Printf(TEXT("[Cartesia][%d] error: %s"), TaskID, *ErrorMsg)); Self->CloseAndRemoveSocket(TaskID, true); return; - } - }); + } }); /*WS->OnRawMessage().AddLambda([WeakThis, TaskID](const void* Data, SIZE_T Size, SIZE_T BytesRemaining) { diff --git a/Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Private/RealtimeAPI/RealtimeAPI_TTSManager.cpp b/Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Private/RealtimeAPI/RealtimeAPI_TTSManager.cpp index 4dca170..9508fce 100644 --- a/Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Private/RealtimeAPI/RealtimeAPI_TTSManager.cpp +++ b/Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Private/RealtimeAPI/RealtimeAPI_TTSManager.cpp @@ -7,4 +7,94 @@ void URealtimeAPI_TTSManager::InitTTSManager(UTTSBaseConfig* InTSSConfig, bool D { UTTSManagerBase::InitTTSManager(InTSSConfig, DebugMode); SetTTSState(ETTSState::Ready); +} + +void URealtimeAPI_TTSManager::AddAudioChunk(const TArray AudioChunk, bool IsFinal) +{ + // Flush any subtitle text that arrived before this first audio chunk created a task. + // Call the base implementation first so the task is created, then update OriginalText. + Super::AddAudioChunk(AudioChunk, IsFinal); + + if (!PendingSubtitleText.IsEmpty()) + { + FScopeLock Lock(&TaskQueueCriticalSection); + if (ActiveTasks.Num() > 0) + { + ActiveTasks.Last().OriginalText += PendingSubtitleText; + PendingSubtitleText.Empty(); + } + } +} + +void URealtimeAPI_TTSManager::AddTextChunk(const FString& TextChunk, bool IsFinal) +{ + // Do not route through the base class: it would either create a TTS generation task + // (bSupportsStreamedInput=false path) or a competing streaming task (true path), + // both of which leave a phantom task in ActiveTasks that blocks playback forever. + bool bShouldSave = false; + bool bCacheHit = false; + FTTSTask TaskSnapshot; + { + FScopeLock Lock(&TaskQueueCriticalSection); + + const FString FullText = PendingSubtitleText + TextChunk; + + if (ActiveTasks.Num() > 0) + { + // AI audio path: a task already exists from AddAudioChunk. + // Update its subtitle text; handle cache save/load when text is final. + FTTSTask& Current = ActiveTasks.Last(); + PendingSubtitleText.Empty(); + Current.OriginalText += FullText; + + if (IsFinal) + { + Current.Text = Current.OriginalText; + if (TTSConfig && TTSConfig->GlobalTTSSettings.UseCacheSystem) + { + if (!Current.bIsGenerating) + { + // Audio already done: save to cache now. + TaskSnapshot = Current; + bShouldSave = true; + } + // else: audio still arriving — OnGeneratedAudioChunkReceived saves + // once it sees Text is non-empty on the last audio chunk. + } + } + } + else if (IsFinal && TTSConfig && TTSConfig->GlobalTTSSettings.UseCacheSystem) + { + // Cache path: no AddAudioChunk was called. Create a task and try to load + // cached audio. If the cache misses, the task is discarded (caller should + // not reach here without cached audio available in pure-cache mode). + PendingSubtitleText.Empty(); + FTTSTask NewTask; + NewTask.TaskID = NextTaskID++; + NewTask.Text = FullText; + NewTask.OriginalText = FullText; + NewTask.bIsGenerating = true; + + if (TryLoadCachedAudioIntoTask(NewTask)) + { + NewTask.bIsGenerating = false; + NewTask.AudioStartByteOffset = 0; + ActiveTasks.Add(NewTask); + bReceivedFinalInput = true; + bCacheHit = true; + } + // Cache miss in pure-cache mode: nothing to play, task discarded. + } + else + { + // Audio task not created yet; buffer until AddAudioChunk creates it. + PendingSubtitleText += TextChunk; + } + } + + if (bShouldSave) + SaveTaskAudioToCache(TaskSnapshot); + + if (bCacheHit) + CheckPlayback(); } \ No newline at end of file diff --git a/Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Private/TTSManagerBase.cpp b/Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Private/TTSManagerBase.cpp index f285228..734e8c5 100644 --- a/Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Private/TTSManagerBase.cpp +++ b/Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Private/TTSManagerBase.cpp @@ -5,8 +5,14 @@ #include "AvatarSoundWave.h" #include "Misc/App.h" #include "Misc/SecureHash.h" +#include "FL_AvatarCoreShared.h" +#include "AvatarCoreSharedEnums.h" #include "Kismet/GameplayStatics.h" #include "HAL/FileManager.h" +#include "Dom/JsonObject.h" +#include "Serialization/JsonSerializer.h" +#include "Serialization/JsonWriter.h" +#include "Serialization/JsonReader.h" UTTSManagerBase::UTTSManagerBase() : RegisteredAudioComponent(nullptr) @@ -294,6 +300,12 @@ void UTTSManagerBase::WipeLastCachedFile() { IFileManager::Get().Delete(*NewestFullPath, /*RequireExists=*/false, /*EvenReadOnly=*/true, /*Quiet=*/true); UE_LOG(LogTemp, Warning, TEXT("File deleted: %s"), *NewestFullPath); + + const FString SidecarPath = FPaths::ChangeExtension(NewestFullPath, TEXT("json")); + if (FPaths::FileExists(SidecarPath)) + { + IFileManager::Get().Delete(*SidecarPath, /*RequireExists=*/false, /*EvenReadOnly=*/true, /*Quiet=*/true); + } } } @@ -306,6 +318,9 @@ void UTTSManagerBase::OnTTSManagerFinished() ProceduralSoundWave->Duration = 0.0f; } + AccumulatedSubtitleText.Empty(); + StreamingInputBuffer.Empty(); + SetTTSState(ETTSState::Ready); } @@ -365,6 +380,8 @@ void UTTSManagerBase::ClearTTS() // Reset state variables bReceivedFinalInput = false; + AccumulatedSubtitleText.Empty(); + StreamingInputBuffer.Empty(); // Set state to Ready SetTTSState(ETTSState::Ready); @@ -378,7 +395,7 @@ void UTTSManagerBase::SetCachingEnabled(bool bEnabled) TTSConfig->GlobalTTSSettings.UseCacheSystem = bEnabled; } -void UTTSManagerBase::SetLanguage(ETTSLanguage NewLanguage) +void UTTSManagerBase::SetTTSLanguage(ELanguage NewLanguage) { if (!TTSConfig) return; TTSConfig->GlobalTTSSettings.Language = NewLanguage; @@ -416,6 +433,59 @@ void UTTSManagerBase::AddTextChunk(const FString& TextChunk, bool IsFinal) if (bShouldStream) { + // Accumulate into buffer and flush only at word boundaries to allow multi-word replacements + StreamingInputBuffer += TextChunk; + + FString ToSend; + if (IsFinal) + { + ToSend = StreamingInputBuffer; + StreamingInputBuffer.Empty(); + } + else + { + // Find last word boundary scanning from the end + int32 LastBoundary = -1; + for (int32 i = StreamingInputBuffer.Len() - 1; i >= 0; --i) + { + TCHAR C = StreamingInputBuffer[i]; + if (C == ' ' || C == '.' || C == ',' || C == '!' || C == '?' || C == ':' || C == ';') + { + LastBoundary = i; + break; + } + } + if (LastBoundary >= 0) + { + ToSend = StreamingInputBuffer.Left(LastBoundary + 1); + StreamingInputBuffer = StreamingInputBuffer.Mid(LastBoundary + 1); + } + } + + // Nothing ready yet: still accumulating mid-word + if (ToSend.IsEmpty()) + { + if (!IsFinal) return; + if (!bHasActiveStreamingTask) return; + // IsFinal + active task with empty ToSend: fall through to close the task + } + + // Apply word replacements to the flushed portion + FString ProcessedChunk = ToSend; + if (TTSConfig && TTSConfig->GlobalTTSSettings.WordReplacements.Num() > 0) + { + for (const FString& Entry : TTSConfig->GlobalTTSSettings.WordReplacements) + { + FString From, To; + if (Entry.Split(TEXT("|"), &From, &To)) + { + From = From.TrimStartAndEnd(); + To = To.TrimStartAndEnd(); + if (!From.IsEmpty()) + ProcessedChunk.ReplaceInline(*From, *To, ESearchCase::IgnoreCase); + } + } + } bool bStartedNewStreamingTask = false; @@ -429,8 +499,8 @@ void UTTSManagerBase::AddTextChunk(const FString& TextChunk, bool IsFinal) { FTTSTask NewTask; NewTask.TaskID = NextTaskID++; - NewTask.OriginalText = TextChunk; - NewTask.Text = TextChunk; + NewTask.OriginalText = ToSend; + NewTask.Text = ProcessedChunk; NewTask.bIsStreaming = !IsFinal; NewTask.bIsGenerating = true; ActiveTasks.Add(NewTask); @@ -440,12 +510,20 @@ void UTTSManagerBase::AddTextChunk(const FString& TextChunk, bool IsFinal) } else { - CurrentTask.OriginalText += TextChunk; - CurrentTask.Text += TextChunk; - if (IsFinal) + // CurrentTask is a local copy; update the actual element in ActiveTasks + for (FTTSTask& ActiveTask : ActiveTasks) + { + if (ActiveTask.TaskID == StreamingTaskID) { - CurrentTask.bIsStreaming = false; + ActiveTask.OriginalText += ToSend; + ActiveTask.Text += ProcessedChunk; + if (IsFinal) + { + ActiveTask.bIsStreaming = false; + } + break; } + } } if (StreamingTaskID != -1) @@ -453,11 +531,11 @@ void UTTSManagerBase::AddTextChunk(const FString& TextChunk, bool IsFinal) if (bStartedNewStreamingTask) { SetTTSState(ETTSState::Producing); - StartStreamingGeneration(StreamingTaskID, TextChunk, IsFinal); + StartStreamingGeneration(StreamingTaskID, ProcessedChunk, IsFinal); } else { - UpdateStreamingText(StreamingTaskID, TextChunk, IsFinal); + UpdateStreamingText(StreamingTaskID, ProcessedChunk, IsFinal); } return; } @@ -547,16 +625,6 @@ void UTTSManagerBase::OnGeneratedAudioChunkReceived(FTTSTask& Task, const TArray ProcessedAudio = AudioData; } - // Run through WebRTC APM for chunk-boundary smoothing (NS + HPF) - if (WebRTCChannel && WebRTCChannel->IsInitialized() && ProcessedAudio.Num() > 0) - { - TArray WebRTCProcessed = WebRTCChannel->ProcessTTSAudio(ProcessedAudio); - if (WebRTCProcessed.Num() > 0) - { - ProcessedAudio = MoveTemp(WebRTCProcessed); - } - } - // Append audio data to the owning ActiveTask buffer, not directly to the output queue FTTSTask* TargetTaskPtr = nullptr; { @@ -577,6 +645,12 @@ void UTTSManagerBase::OnGeneratedAudioChunkReceived(FTTSTask& Task, const TArray return; } + // Record where speech audio starts (before the first real chunk is appended) + if (TargetTaskPtr->AudioStartByteOffset < 0 && ProcessedAudio.Num() > 0) + { + TargetTaskPtr->AudioStartByteOffset = TargetTaskPtr->AudioData.Num(); + } + // Append first TargetTaskPtr->AudioData.Append(ProcessedAudio); @@ -604,9 +678,16 @@ void UTTSManagerBase::OnGeneratedAudioChunkReceived(FTTSTask& Task, const TArray TargetTaskPtr->bIsComplete = true; } } - if (IsLastChunk && TTSConfig->GlobalTTSSettings.UseCacheSystem && !bIsPreCaching) + if (IsLastChunk && TTSConfig->GlobalTTSSettings.UseCacheSystem && !bIsPreCaching && TargetTaskPtr && !TargetTaskPtr->Text.IsEmpty()) { - SaveTaskAudioToCache(Task); + // Use TargetTaskPtr (actual task) not Task (may be a stale local copy from AddAudioChunk). + // TargetTaskPtr has the complete AudioData including the last chunk just appended above, + // and the correct Text field. Access outside the lock is safe: bIsGenerating=false means + // no further audio appends can occur on this task. + // Note: for RealtimeAPI, Text may still be empty here if the final text chunk has not + // arrived yet — in that case, RealtimeAPI_TTSManager::AddTextChunk triggers the save + // once both streams are finalised. + SaveTaskAudioToCache(*TargetTaskPtr); } UTTSManagerBase::CheckPlayback(); } @@ -739,6 +820,22 @@ void UTTSManagerBase::OnAvatarSoundwaveBufferUnderun() SetTTSState(ETTSState::WaitingForChunks); TTSLog(TEXT("Stopping AudioComponent playback")); RegisteredAudioComponent->Stop(); + + // Stop() wipes the ProceduralSoundWave internal buffer. Reset QueueCursor to + // PlaybackCursor so those bytes can be re-queued by FlushToAudioQueue. + // Clear QueuedSegments since the audio device will no longer render those bytes. + { + FScopeLock TaskLock(&TaskQueueCriticalSection); + for (FTTSTask& Task : ActiveTasks) + { + Task.QueueCursor = Task.PlaybackCursor; + } + } + { + FScopeLock SegLock(&SegmentQueueCriticalSection); + QueuedSegments.Empty(); + } + if (WebRTCChannel) WebRTCChannel->Reset(); } } @@ -909,18 +1006,42 @@ bool UTTSManagerBase::FlushToAudioQueue() if (ProceduralSoundWave) { TTSLog(FString::Printf(TEXT("Queueing %d bytes for task %d"), BytesToQueue, Curr.TaskID)); - // Queue the available tail for this task - ProceduralSoundWave->QueueAudio(Curr.AudioData.GetData() + BytesQueuedSoFar, BytesToQueue); + // Apply WebRTC HPF smoothing for live playback; Task.AudioData stays raw for clean caching + TArray AudioToQueue(Curr.AudioData.GetData() + BytesQueuedSoFar, BytesToQueue); + if (WebRTCChannel && WebRTCChannel->IsInitialized()) + { + TArray Processed = WebRTCChannel->ProcessTTSAudio(AudioToQueue); + if (Processed.Num() > 0) + AudioToQueue = MoveTemp(Processed); + } + ProceduralSoundWave->QueueAudio(AudioToQueue.GetData(), AudioToQueue.Num()); // Update duration accurately: seconds = bytes / (bytesPerSample * channels * sampleRate) - const float DurationInc = static_cast(BytesToQueue) / static_cast(BytesPerSecond); + const float DurationInc = static_cast(AudioToQueue.Num()) / static_cast(BytesPerSecond); ProceduralSoundWave->Duration += DurationInc; + // Drain WebRTC accumulator on the final segment so no audio is held back. + // Use !bIsGenerating (not bIsComplete) — bIsComplete requires PlaybackCursor to catch up, + // which hasn't happened yet. bIsGenerating=false means the API sent all chunks. + if (!Curr.bIsGenerating && (Curr.QueueCursor + BytesToQueue >= Curr.AudioData.Num())) + { + if (WebRTCChannel && WebRTCChannel->IsInitialized()) + { + TArray Tail = WebRTCChannel->Flush(); + if (Tail.Num() > 0) + { + ProceduralSoundWave->QueueAudio(Tail.GetData(), Tail.Num()); + ProceduralSoundWave->Duration += static_cast(Tail.Num()) / static_cast(BytesPerSecond); + } + } + } } - // Advance queue cursor but DO NOT mark as rendered yet + // Capture offset before advancing the cursor so ConsumeRenderedBytes can map bytes→time + const int32 SegmentStartOffset = Curr.QueueCursor; + // Advance by AudioData bytes (keeps PlaybackCursor in AudioData coordinate space for subtitle timing) Curr.QueueCursor += BytesToQueue; // Track this queued segment for precise playback accounting { FScopeLock SegLock(&SegmentQueueCriticalSection); - QueuedSegments.Add({ Curr.TaskID, BytesToQueue }); + QueuedSegments.Add({ Curr.TaskID, BytesToQueue, SegmentStartOffset }); } bQueuedAny = true; @@ -1103,8 +1224,7 @@ void UTTSManagerBase::ProcessText(bool IsFinal) } if (IsFinal && !Text.IsEmpty()) { - FString Sentence = Text; - Sentence = ProcessTextForGeneration(Sentence); + const FString Sentence = Text; if (!Sentence.IsEmpty()) { // Directly add to PendingTasks queue instead of redundant SentenceQueue @@ -1280,10 +1400,21 @@ FString UTTSManagerBase::GetCacheDirectory() const FString UTTSManagerBase::GetCacheFilePath(const FString& InText) const { - const FString Hash = ComputeTaskHash(InText); + UE_LOG(LogTemp, Warning, TEXT("InText is empty therefore hash cannot be created, using invalidCacheNoText instead.")); + FString Hash; + if (InText.IsEmpty()) + Hash = "invalidCacheNoText"; + else + Hash = ComputeTaskHash(InText); return FPaths::Combine(GetCacheDirectory(), Hash + TEXT(".wav")); } +FString UTTSManagerBase::GetCacheSidecarPath(const FString& InText) const +{ + const FString Hash = ComputeTaskHash(InText); + return FPaths::Combine(GetCacheDirectory(), Hash + TEXT(".json")); +} + bool UTTSManagerBase::HasCacheFile(const FString& InText) { const FString FilePath = GetCacheFilePath(ProcessTextForGeneration(InText)); @@ -1380,6 +1511,32 @@ bool UTTSManagerBase::SaveTaskAudioToCache(const FTTSTask& Task) if (bOk) { TTSLog(FString::Printf(TEXT("Saved cache: %s (%d bytes)"), *FilePath, Wav.Num())); + + if (Task.WordTimestamps.Num() > 0) + { + TSharedRef Root = MakeShared(); + TArray> WordsArray; + for (const FWordTimestamp& WT : Task.WordTimestamps) + { + TSharedRef Entry = MakeShared(); + Entry->SetStringField(TEXT("word"), WT.Word); + Entry->SetNumberField(TEXT("start"), WT.Start); + Entry->SetNumberField(TEXT("end"), WT.End); + WordsArray.Add(MakeShared(Entry)); + } + Root->SetArrayField(TEXT("words"), WordsArray); + + FString JsonStr; + TSharedRef> Writer = TJsonWriterFactory<>::Create(&JsonStr); + FJsonSerializer::Serialize(Root, Writer); + + const FString SidecarPath = GetCacheSidecarPath(Task.Text); + + if (!FFileHelper::SaveStringToFile(JsonStr, *SidecarPath)) + { + TTSLog(FString::Printf(TEXT("Failed to save timestamp sidecar: %s"), *SidecarPath)); + } + } } else { @@ -1413,6 +1570,37 @@ bool UTTSManagerBase::TryLoadCachedAudioIntoTask(FTTSTask& Task) Task.AudioData.Append(DataPtr, DataSize); IFileManager::Get().SetTimeStamp(*FilePath, FDateTime::Now()); + + const FString SidecarPath = GetCacheSidecarPath(Task.Text); + if (FPaths::FileExists(SidecarPath)) + { + FString JsonStr; + if (FFileHelper::LoadFileToString(JsonStr, *SidecarPath)) + { + TSharedPtr Root; + TSharedRef> Reader = TJsonReaderFactory<>::Create(JsonStr); + if (FJsonSerializer::Deserialize(Reader, Root) && Root.IsValid()) + { + const TArray>* WordsArr; + if (Root->TryGetArrayField(TEXT("words"), WordsArr)) + { + for (const TSharedPtr& Val : *WordsArr) + { + const TSharedPtr* Obj; + if (Val->TryGetObject(Obj)) + { + FWordTimestamp WT; + (*Obj)->TryGetStringField(TEXT("word"), WT.Word); + (*Obj)->TryGetNumberField(TEXT("start"), WT.Start); + (*Obj)->TryGetNumberField(TEXT("end"), WT.End); + Task.WordTimestamps.Add(WT); + } + } + } + } + } + } + return true; } @@ -1421,6 +1609,15 @@ void UTTSManagerBase::ConsumeRenderedBytes(int32 Bytes) if (Bytes <= 0) return; + if (!TTSConfig) + return; + + const int32 BytesPerSample = 2; // 16-bit PCM + const int32 EffSR = (TTSConfig->GlobalTTSSettings.ResampleToSampleRate > 0) + ? TTSConfig->GlobalTTSSettings.ResampleToSampleRate + : TTSConfig->GlobalTTSSettings.AudioSampleRate; + const int32 BytesPerSecond = EffSR * TTSConfig->GlobalTTSSettings.AudioNumChannels * BytesPerSample; + // Drain the queued segment FIFO while updating per-task playback // Lock order: SegmentQueue first, then TaskQueue to avoid deadlock with FlushToAudioQueue FScopeLock SegLock(&SegmentQueueCriticalSection); @@ -1450,6 +1647,84 @@ void UTTSManagerBase::ConsumeRenderedBytes(int32 Bytes) { Task.bIsComplete = true; } + + // --- Subtitle event --- + TArray OrigWords; + Task.OriginalText.ParseIntoArray(OrigWords, TEXT(" "), true); + if (OrigWords.Num() > 0) + { + int32 NewWordIdx = Task.LastSubtitleWordIndex; + + // TimeNow is relative to the start of Cartesia's audio (not the task buffer start), + // so subtract any silence that was prepended before the first real chunk. + const int32 SpeechStart = (Task.AudioStartByteOffset >= 0) ? Task.AudioStartByteOffset : 0; + const int32 SpeechCursor = FMath::Max(Task.PlaybackCursor - SpeechStart, 0); + const int32 SpeechTotal = FMath::Max(Task.AudioData.Num() - SpeechStart, 0); + + if (Task.WordTimestamps.Num() > 0 && BytesPerSecond > 0) + { + // Timestamp path: advance at most one word per callback so timestamps that + // arrived before audio started playing don't all fire in one burst. + const float TimeNow = (float)SpeechCursor / (float)BytesPerSecond; + const int32 NextWi = Task.LastSubtitleWordIndex + 1; + if (NextWi < Task.WordTimestamps.Num() && Task.WordTimestamps[NextWi].Start <= TimeNow) + { + NewWordIdx = NextWi; + } + } + else + { + // Fallback: proportional interpolation, also one word at a time + const float Fraction = (SpeechTotal > 0) + ? FMath::Clamp((float)SpeechCursor / (float)SpeechTotal, 0.f, 1.f) + : 0.f; + const int32 ProportionalIdx = FMath::Clamp( + FMath::FloorToInt(Fraction * OrigWords.Num()) - 1, + Task.LastSubtitleWordIndex, OrigWords.Num() - 1); + if (ProportionalIdx > Task.LastSubtitleWordIndex) + NewWordIdx = Task.LastSubtitleWordIndex + 1; + } + + if (NewWordIdx > Task.LastSubtitleWordIndex) + { + FString ChunkText; + for (int32 wi = Task.LastSubtitleWordIndex + 1; wi <= NewWordIdx; ++wi) + { + if (OrigWords.IsValidIndex(wi)) + { + if (!ChunkText.IsEmpty()) ChunkText += TEXT(" "); + ChunkText += OrigWords[wi]; + } + } + Task.LastSubtitleWordIndex = NewWordIdx; + + if (!ChunkText.IsEmpty()) + { + AccumulatedSubtitleText += (AccumulatedSubtitleText.IsEmpty() ? TEXT("") : TEXT(" ")) + ChunkText; + } + + const bool bIsFinal = Task.bIsComplete + && Task.PlaybackCursor >= Task.AudioData.Num() + && QueuedSegments.Num() <= 1; + + if (!ChunkText.IsEmpty() || bIsFinal) + { + const int32 FireTaskID = Task.TaskID; + FString ChunkCopy = ChunkText; + FString AccumCopy = AccumulatedSubtitleText; + TWeakObjectPtr WeakThis(this); + AsyncTask(ENamedThreads::GameThread, [WeakThis, FireTaskID, ChunkCopy, AccumCopy, bIsFinal]() + { + if (WeakThis.IsValid()) + { + WeakThis->OnTTSSubtitle.Broadcast(FireTaskID, ChunkCopy, AccumCopy, bIsFinal); + } + }); + } + } + } + // --- End subtitle event --- + break; } } diff --git a/Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Private/TTSWebRTCChannel.cpp b/Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Private/TTSWebRTCChannel.cpp index 8dd9a53..95c47a0 100644 --- a/Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Private/TTSWebRTCChannel.cpp +++ b/Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Private/TTSWebRTCChannel.cpp @@ -53,9 +53,8 @@ void FTTSWebRTCChannel::Configure() Config.echo_canceller.enabled = false; // AEC handled by AvatarCore_STT Config.pre_amplifier.enabled = false; Config.high_pass_filter.enabled = true; // removes DC offset / low-freq rumble - Config.noise_suppression.enabled = true; // smooths inter-chunk artifacts - Config.noise_suppression.level = webrtc::AudioProcessing::Config::NoiseSuppression::kLow; - Config.transient_suppression.enabled = true; // suppresses click artifacts at chunk boundaries + Config.noise_suppression.enabled = false; // causes artifacts on clean TTS audio + Config.transient_suppression.enabled = false; // designed for mic capture, not synthesis Config.gain_controller1.enabled = false; Config.gain_controller2.enabled = false; @@ -79,6 +78,36 @@ void FTTSWebRTCChannel::Reset() } } +// --------------------------------------------------------------------------- +TArray FTTSWebRTCChannel::Flush() +{ + FScopeLock Lock(&CriticalSection); + if (Accumulator.Num() == 0 || !bInitialized || !AudioProcessing || FrameSamplesTotal <= 0) + return {}; + + // Zero-pad to a complete frame + while (Accumulator.Num() < FrameSamplesTotal) + Accumulator.Add(0); + + webrtc::StreamConfig StreamCfg(WebRTCRate, NumChannels); + TArray FrameBuf; + FrameBuf.SetNumUninitialized(FrameSamplesTotal); + FMemory::Memcpy(FrameBuf.GetData(), Accumulator.GetData(), FrameSamplesTotal * sizeof(int16)); + AudioProcessing->ProcessStream(FrameBuf.GetData(), StreamCfg, StreamCfg, FrameBuf.GetData()); + Accumulator.Reset(); + + TArray FinalInt16; + if (SampleRate != WebRTCRate) + ResampleInt16Linear(FrameBuf, WebRTCRate, SampleRate, FinalInt16); + else + FinalInt16 = MoveTemp(FrameBuf); + + TArray Out; + Out.SetNumUninitialized(FinalInt16.Num() * 2); + FMemory::Memcpy(Out.GetData(), FinalInt16.GetData(), Out.Num()); + return Out; +} + // --------------------------------------------------------------------------- TArray FTTSWebRTCChannel::ProcessTTSAudio(const TArray& InPCM16) { diff --git a/Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Public/RealtimeAPI/RealtimeAPI_TTSManager.h b/Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Public/RealtimeAPI/RealtimeAPI_TTSManager.h index ac95fce..f1c670c 100644 --- a/Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Public/RealtimeAPI/RealtimeAPI_TTSManager.h +++ b/Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Public/RealtimeAPI/RealtimeAPI_TTSManager.h @@ -17,5 +17,14 @@ class AVATARCORE_TTS_API URealtimeAPI_TTSManager : public UTTSManagerBase public: // UTTSManagerBase overrides virtual void InitTTSManager(UTTSBaseConfig* InTSSConfig, bool DebugMode) override; - + + // In the RealtimeAPI flow, audio comes directly via AddAudioChunk while text is piped + // in separately for subtitle display. These overrides keep text and audio on the same + // task without creating additional TTS generation tasks. + virtual void AddTextChunk(const FString& TextChunk, bool IsFinal) override; + virtual void AddAudioChunk(const TArray AudioChunk, bool IsFinal) override; + +private: + // Buffer for text that arrives before the first audio chunk creates a task + FString PendingSubtitleText; }; diff --git a/Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Public/TTSBaseConfig.h b/Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Public/TTSBaseConfig.h index a058fd9..07a836c 100644 --- a/Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Public/TTSBaseConfig.h +++ b/Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Public/TTSBaseConfig.h @@ -4,6 +4,7 @@ #include "CoreMinimal.h" #include "UObject/NoExportTypes.h" +#include "AvatarCoreSharedEnums.h" #include "TTSBaseConfig.generated.h" UENUM(BlueprintType) @@ -14,76 +15,6 @@ enum class ECommaSplitRule : uint8 { FillWord UMETA(DisplayName = "Split on every comma, but add fill words.") }; -UENUM(BlueprintType) -enum class ETTSKeepAliveRule : uint8 { - EditorOnly UMETA(DisplayName = "Only keep TTS Module open in Editor Mode"), - Never UMETA(DisplayName = "Never keep TTS Module alive."), - Always UMETA(DisplayName = "Keep TTS Module alive in Editor and Shipping mode.") -}; - -UENUM(BlueprintType) -enum class ETTSLanguage : uint8 -{ - NONE UMETA(DisplayName = "Unset"), - en UMETA(DisplayName = "English"), - fr UMETA(DisplayName = "French"), - de UMETA(DisplayName = "German"), - es UMETA(DisplayName = "Spanish"), - pt UMETA(DisplayName = "Portuguese"), - zh UMETA(DisplayName = "Chinese"), - ja UMETA(DisplayName = "Japanese"), - hi UMETA(DisplayName = "Hindi"), - it UMETA(DisplayName = "Italian"), - ko UMETA(DisplayName = "Korean"), - nl UMETA(DisplayName = "Dutch"), - pl UMETA(DisplayName = "Polish"), - ru UMETA(DisplayName = "Russian"), - sv UMETA(DisplayName = "Swedish"), - tr UMETA(DisplayName = "Turkish"), - tl UMETA(DisplayName = "Filipino"), - bg UMETA(DisplayName = "Bulgarian"), - ro UMETA(DisplayName = "Romanian"), - ar UMETA(DisplayName = "Arabic"), - cs UMETA(DisplayName = "Czech"), - el UMETA(DisplayName = "Greek"), - fi UMETA(DisplayName = "Finnish"), - hr UMETA(DisplayName = "Croatian"), - ms UMETA(DisplayName = "Malay"), - sk UMETA(DisplayName = "Slovak"), - da UMETA(DisplayName = "Danish"), - ta UMETA(DisplayName = "Tamil"), - uk UMETA(DisplayName = "Ukrainian"), - hu UMETA(DisplayName = "Hungarian"), - no UMETA(DisplayName = "Norwegian"), - vi UMETA(DisplayName = "Vietnamese"), - bn UMETA(DisplayName = "Bengali"), - th UMETA(DisplayName = "Thai"), - he UMETA(DisplayName = "Hebrew"), - ka UMETA(DisplayName = "Georgian"), - id UMETA(DisplayName = "Indonesian"), - te UMETA(DisplayName = "Telugu"), - gu UMETA(DisplayName = "Gujarati"), - kn UMETA(DisplayName = "Kannada"), - ml UMETA(DisplayName = "Malayalam"), - mr UMETA(DisplayName = "Marathi"), - pa UMETA(DisplayName = "Punjabi"), -}; - -// Returns the ISO 639-1 code string for the given language enum (e.g. ETTSLanguage::de → "de"). -// Derived from the enum value name via UE reflection, so it stays in sync automatically. -static FORCEINLINE FString TTSLanguageToString(ETTSLanguage Language) -{ - FString Name = StaticEnum()->GetNameStringByValue(static_cast(Language)); - // StaticEnum returns the short name (no "ETTSLanguage::" prefix in UE5 GetNameStringByValue) - // but strip the prefix defensively in case the runtime returns the qualified form. - int32 ScopePos; - if (Name.FindLastChar(TEXT(':'), ScopePos)) - { - Name = Name.RightChop(ScopePos + 1); - } - return Name.IsEmpty() ? TEXT("en") : Name; -} - USTRUCT(BlueprintType) struct FGlobalTTSSettings { @@ -134,7 +65,7 @@ struct FGlobalTTSSettings int32 MaxCharacterForGeneration = 0; UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreTTS|Base", meta = (ExposeOnSpawn = "true")) - ETTSLanguage Language = ETTSLanguage::de; + ELanguage Language = ELanguage::de; }; class UTTSManagerBase; diff --git a/Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Public/TTSManagerBase.h b/Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Public/TTSManagerBase.h index 813059e..a43a5f3 100644 --- a/Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Public/TTSManagerBase.h +++ b/Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Public/TTSManagerBase.h @@ -29,6 +29,15 @@ enum class ETTSState : uint8 FinishPlayback UMETA(DisplayName = "Finish Playback") }; +USTRUCT() +struct FWordTimestamp +{ + GENERATED_BODY() + FString Word; + float Start = 0.f; + float End = 0.f; +}; + // TTS Task struct for concurrent generation USTRUCT(BlueprintType) struct FTTSTask @@ -48,6 +57,9 @@ struct FTTSTask // Queued cursor: how many bytes have been queued to the procedural wave (may be ahead of PlaybackCursor) int32 QueueCursor = 0; bool bIsComplete = false; // True when all chunks are received and played + TArray WordTimestamps; // populated async by Cartesia timestamp messages + int32 LastSubtitleWordIndex = -1; // last word index fired in TTSSubtitle event + int32 AudioStartByteOffset = -1; // byte offset in AudioData where first real audio chunk begins (after any prepended silence) FTTSTask() : Text(), OriginalText(), TaskID(0) {} FTTSTask(const FString& InText, FString& InOriginalText, int32 InTaskID) : Text(InText), OriginalText(InOriginalText), TaskID(InTaskID) {} }; @@ -58,6 +70,7 @@ DECLARE_DYNAMIC_MULTICAST_DELEGATE(FOnTTSCacheFinished); DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FOnTTSError, const FString&, ErrorMessage); DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FOnTTSStateChanged, ETTSState, NewState); DECLARE_DYNAMIC_MULTICAST_DELEGATE_ThreeParams(FOnTTSAudioChunkForBP, const TArray&, PCMData, int32, SampleRate, int32, NumChannels); +DECLARE_DYNAMIC_MULTICAST_DELEGATE_FourParams(FOnTTSSubtitle, int32, TaskID, const FString&, CurrentChunkText, const FString&, AccumulatedSubtitle, bool, bIsFinal); UCLASS(Abstract, Blueprintable, BlueprintType) class AVATARCORE_TTS_API UTTSManagerBase : public UObject @@ -92,6 +105,10 @@ public: UPROPERTY(BlueprintAssignable, Category = "AvatarCoreTTS|Events") FOnTTSCacheUpdate TTSCacheUpdate; + // Fired as words are spoken; CurrentChunkText = words spoken since last event, AccumulatedSubtitle = all words spoken so far + UPROPERTY(BlueprintAssignable, Category = "AvatarCoreTTS|Events") + FOnTTSSubtitle OnTTSSubtitle; + // Initializes the Manager and set Config file UFUNCTION(BlueprintCallable, Category = "AvatarCoreTTS|Lifecycle") virtual void InitTTSManager(UTTSBaseConfig* InTSSConfig, bool DebugMode); @@ -109,7 +126,7 @@ public: // Set the language used for TTS generation UFUNCTION(BlueprintCallable, Category = "AvatarCoreTTS|Settings") - void SetLanguage(ETTSLanguage NewLanguage); + void SetTTSLanguage(ELanguage NewLanguage); // Is the TTSManager currently busy UFUNCTION(BlueprintCallable, Category = "AvatarCoreTTS|Helper") @@ -217,8 +234,6 @@ protected: void SetTTSState(ETTSState NewState); - void AddEndSilence(FTTSTask& Task); - // Sentence segmentation FString PendingText; FCriticalSection PendingTextCriticalSection; @@ -244,9 +259,6 @@ protected: virtual void AddToRootManager(); virtual void RemoveFromRootManager(); - // Centralized Audio Queue Management - void AddSilenceToTask(FTTSTask& Task, float SilenceLength); - bool bIsRooted; // Timers @@ -281,6 +293,9 @@ private: TArray PreCacheSentences; + // Accumulates streaming text until a word boundary before applying replacements and forwarding + FString StreamingInputBuffer; + TUniquePtr WebRTCChannel; // Timer for polling Ready transition after playback finished @@ -294,17 +309,20 @@ protected: FString GetCacheDirectory() const; // Full cache file path for given text (Hash.wav) FString GetCacheFilePath(const FString& InText) const; + // Sidecar JSON path for word timestamps (Hash.json, same stem as wav) + FString GetCacheSidecarPath(const FString& InText) const; // Try to load cached WAV into Task.AudioData (returns true on success) bool TryLoadCachedAudioIntoTask(FTTSTask& Task); // Save Task.AudioData as WAV to cache (returns true on success) bool SaveTaskAudioToCache(const FTTSTask& Task); - bool bInitialSilenceDone = false; - void OnGeneratedAudioChunkReceived(FTTSTask& Task, const TArray& AudioData, bool IsLastChunk); + // Accumulated subtitle text across the current speech session; cleared by ClearTTS + FString AccumulatedSubtitleText; + // Accurate playback accounting without relying on underflow: FIFO of queued segments - struct FQueuedSegment { int32 TaskID; int32 BytesRemaining; }; + struct FQueuedSegment { int32 TaskID; int32 BytesRemaining; int32 StartByteOffset; }; FCriticalSection SegmentQueueCriticalSection; TArray QueuedSegments; // treated as a queue (pop from front) diff --git a/Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Public/TTSWebRTCChannel.h b/Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Public/TTSWebRTCChannel.h index c8a8776..807577c 100644 --- a/Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Public/TTSWebRTCChannel.h +++ b/Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Public/TTSWebRTCChannel.h @@ -41,6 +41,9 @@ public: /** Flush and discard any accumulated partial frame. */ void Reset(); + /** Zero-pad accumulator to a full frame, process it, return the result, then clear the accumulator. Returns empty if accumulator is empty. */ + TArray Flush(); + bool IsInitialized() const { return bInitialized; } private: diff --git a/Unreal/Plugins/BLogger/Source/BLogger/Public/SLogWidgetState.h b/Unreal/Plugins/BLogger/Source/BLogger/Public/SLogWidgetState.h index 71804ba..860f660 100644 --- a/Unreal/Plugins/BLogger/Source/BLogger/Public/SLogWidgetState.h +++ b/Unreal/Plugins/BLogger/Source/BLogger/Public/SLogWidgetState.h @@ -15,7 +15,7 @@ private: bool IsScrollToBottomEnabled = true; bool IsAutoRefreshEnabled = true; TArray FilteredCategories; - int32 VerbosityFilterMask = 127; + int32 VerbosityFilterMask = 126; // VeryVerbose (1) excluded by default FString SearchText; public: diff --git a/Unreal/Plugins/BTools/Content/Components/CheckProcessComponent.uasset b/Unreal/Plugins/BTools/Content/Components/CheckProcessComponent.uasset new file mode 100644 index 0000000..62bc43d --- /dev/null +++ b/Unreal/Plugins/BTools/Content/Components/CheckProcessComponent.uasset @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6c90e936133e20745038f7d327f72f0826a57618208d81aecaea3e428f390840 +size 89515 diff --git a/Unreal/Plugins/BTools/Source/BTools/Private/BToolsBPLibrary.cpp b/Unreal/Plugins/BTools/Source/BTools/Private/BToolsBPLibrary.cpp index bd81e9e..67a8eab 100644 --- a/Unreal/Plugins/BTools/Source/BTools/Private/BToolsBPLibrary.cpp +++ b/Unreal/Plugins/BTools/Source/BTools/Private/BToolsBPLibrary.cpp @@ -15,24 +15,24 @@ #include "BToolsBPLibrary.h" #if PLATFORM_WINDOWS - #include - #pragma comment(lib, "ws2_32.lib") - #include - #include "Misc/Paths.h" - #include - #pragma comment(lib, "iphlpapi.lib") - #include - #include "GenericPlatform/GenericWindow.h" - #include +#include +#pragma comment(lib, "ws2_32.lib") +#include +#include "Misc/Paths.h" +#include +#pragma comment(lib, "iphlpapi.lib") +#include +#include "GenericPlatform/GenericWindow.h" +#include //Create Shortscuts on Runtime - #include "Windows/AllowWindowsPlatformTypes.h" - #include "Windows/WindowsHWrapper.h" - #include - #include - #include "Windows/HideWindowsPlatformTypes.h" +#include "Windows/AllowWindowsPlatformTypes.h" +#include "Windows/WindowsHWrapper.h" +#include +#include +#include "Windows/HideWindowsPlatformTypes.h" //For getting Desktop Path - #include - #include +#include +#include #endif #include "Components/NamedSlot.h" #include "Runtime/UMG/public/Blueprint/UserWidget.h" @@ -119,8 +119,12 @@ bool UBToolsBPLibrary::KillProcessByName(FString Name) #endif } -int UBToolsBPLibrary::KillProcessesByPath(FString Dir) +int UBToolsBPLibrary::KillProcessesByPath(FString Dir, bool UseAbsolutePath) { + + if (Dir.IsEmpty()) + return 0; + #if PLATFORM_WINDOWS int32 ReturnCode = 0; FString Out, Err; int32 Killed = 0; @@ -139,28 +143,37 @@ int UBToolsBPLibrary::KillProcessesByPath(FString Dir) PROCESSENTRY32 Entry; Entry.dwSize = sizeof(PROCESSENTRY32); if (::Process32First(SnapShot, &Entry)) { - + do { - if (FCString::Stricmp(TEXT("python.exe"), Entry.szExeFile) == 0) + HANDLE ProcessHandle = ::OpenProcess(PROCESS_TERMINATE | PROCESS_QUERY_LIMITED_INFORMATION, /*bInherit*/ false, Entry.th32ProcessID); + if (ProcessHandle) { - HANDLE ProcessHandle = ::OpenProcess(PROCESS_TERMINATE | PROCESS_QUERY_LIMITED_INFORMATION, /*bInherit*/ false, Entry.th32ProcessID); - if (ProcessHandle) + WCHAR Buffer[1024]; DWORD Size = UE_ARRAY_COUNT(Buffer); + if (::QueryFullProcessImageNameW(ProcessHandle, 0, Buffer, &Size)) { - WCHAR Buffer[1024]; DWORD Size = UE_ARRAY_COUNT(Buffer); - if (::QueryFullProcessImageNameW(ProcessHandle, 0, Buffer, &Size)) + FString ProcPath(Buffer); + ProcPath.ReplaceInline(TEXT("/"), TEXT("\\")); + // Case-insensitive prefix check + if (UseAbsolutePath) { - FString ProcPath(Buffer); - ProcPath.ReplaceInline(TEXT("/"), TEXT("\\")); - // Case-insensitive prefix check if (ProcPath.Len() >= ScriptsDirFull.Len() && ProcPath.Left(ScriptsDirFull.Len()).Compare(ScriptsDirFull, ESearchCase::IgnoreCase) == 0) { ::TerminateProcess(ProcessHandle, 0); ++Killed; } } - ::CloseHandle(ProcessHandle); + else //Only contains string + { + if (ProcPath.Contains(Dir, ESearchCase::Type::IgnoreCase, ESearchDir::FromStart)) + { + ::TerminateProcess(ProcessHandle, 0); + ++Killed; + } + } + } + ::CloseHandle(ProcessHandle); } } while (::Process32Next(SnapShot, &Entry)); } @@ -810,9 +823,9 @@ void UBToolsBPLibrary::IcmpPingAsync(const FString& DomainName, FOnPingComplete { UE_LOG(LogTemp, Error, TEXT("Failed to resolve domain name: %s"), *DomainName); AsyncTask(ENamedThreads::GameThread, [CompletionCallback]() - { - CompletionCallback.ExecuteIfBound(false, 0.0f); - }); + { + CompletionCallback.ExecuteIfBound(false, 0.0f); + }); return; } @@ -827,9 +840,9 @@ void UBToolsBPLibrary::IcmpPingAsync(const FString& DomainName, FOnPingComplete { UE_LOG(LogTemp, Error, TEXT("Unable to open ICMP handle")); AsyncTask(ENamedThreads::GameThread, [CompletionCallback]() - { - CompletionCallback.ExecuteIfBound(false, 0.0f); - }); return; + { + CompletionCallback.ExecuteIfBound(false, 0.0f); + }); return; } char SendData[] = "PingData"; @@ -840,9 +853,9 @@ void UBToolsBPLibrary::IcmpPingAsync(const FString& DomainName, FOnPingComplete UE_LOG(LogTemp, Error, TEXT("Unable to allocate memory for reply buffer")); IcmpCloseHandle(hIcmpFile); AsyncTask(ENamedThreads::GameThread, [CompletionCallback]() - { - CompletionCallback.ExecuteIfBound(false, 0.0f); - }); return; + { + CompletionCallback.ExecuteIfBound(false, 0.0f); + }); return; } // Send the ICMP echo request @@ -854,24 +867,24 @@ void UBToolsBPLibrary::IcmpPingAsync(const FString& DomainName, FOnPingComplete //UE_LOG(LogTemp, Log, TEXT("Sent ICMP message to %s"), *DomainName); //UE_LOG(LogTemp, Log, TEXT("Received %ld bytes from %s in %ld ms"), pEchoReply->DataSize, *DomainName, PingTime); AsyncTask(ENamedThreads::GameThread, [CompletionCallback, PingTime]() - { - CompletionCallback.ExecuteIfBound(true, PingTime); - }); + { + CompletionCallback.ExecuteIfBound(true, PingTime); + }); } else { UE_LOG(LogTemp, Error, TEXT("Ping to %s failed"), *DomainName); AsyncTask(ENamedThreads::GameThread, [CompletionCallback]() - { - CompletionCallback.ExecuteIfBound(false, 0.0f); - }); + { + CompletionCallback.ExecuteIfBound(false, 0.0f); + }); } // Cleanup free(ReplyBuffer); IcmpCloseHandle(hIcmpFile); }); - #endif +#endif } @@ -1069,36 +1082,39 @@ BOOL CALLBACK MonitorRectProc(HMONITOR hMonitor, HDC hdcMonitor, LPRECT lprcMoni void UBToolsBPLibrary::BringAppToForeground() { #if PLATFORM_WINDOWS - // Try to get the main Unreal window. If not available, fallback to first top-level window. - TSharedPtr MainWindow = FSlateApplication::IsInitialized() ? FSlateApplication::Get().GetActiveTopLevelWindow() : nullptr; - if (!MainWindow.IsValid()) { - const TArray>& TopWindows = FSlateApplication::Get().GetTopLevelWindows(); - if (TopWindows.Num() > 0) { - MainWindow = TopWindows[0]; - } - } - if (MainWindow.IsValid()) { - TSharedPtr NativeWindow = MainWindow->GetNativeWindow(); - if (NativeWindow.IsValid()) { - HWND hWnd = static_cast(NativeWindow->GetOSWindowHandle()); - if (hWnd) { - // Attach input to bypass focus restrictions - DWORD CurrentThreadId = GetCurrentThreadId(); - DWORD ForegroundThreadId = GetWindowThreadProcessId(GetForegroundWindow(), nullptr); - AttachThreadInput(ForegroundThreadId, CurrentThreadId, true); - SetForegroundWindow(hWnd); - SetFocus(hWnd); - SetActiveWindow(hWnd); - AttachThreadInput(ForegroundThreadId, CurrentThreadId, false); - } else { - UE_LOG(LogTemp, Warning, TEXT("BringAppToForeground: hWnd is null")); - } - } else { - UE_LOG(LogTemp, Warning, TEXT("BringAppToForeground: NativeWindow invalid")); - } - } else { - UE_LOG(LogTemp, Warning, TEXT("BringAppToForeground: MainWindow invalid")); - } + // Try to get the main Unreal window. If not available, fallback to first top-level window. + TSharedPtr MainWindow = FSlateApplication::IsInitialized() ? FSlateApplication::Get().GetActiveTopLevelWindow() : nullptr; + if (!MainWindow.IsValid()) { + const TArray>& TopWindows = FSlateApplication::Get().GetTopLevelWindows(); + if (TopWindows.Num() > 0) { + MainWindow = TopWindows[0]; + } + } + if (MainWindow.IsValid()) { + TSharedPtr NativeWindow = MainWindow->GetNativeWindow(); + if (NativeWindow.IsValid()) { + HWND hWnd = static_cast(NativeWindow->GetOSWindowHandle()); + if (hWnd) { + // Attach input to bypass focus restrictions + DWORD CurrentThreadId = GetCurrentThreadId(); + DWORD ForegroundThreadId = GetWindowThreadProcessId(GetForegroundWindow(), nullptr); + AttachThreadInput(ForegroundThreadId, CurrentThreadId, true); + SetForegroundWindow(hWnd); + SetFocus(hWnd); + SetActiveWindow(hWnd); + AttachThreadInput(ForegroundThreadId, CurrentThreadId, false); + } + else { + UE_LOG(LogTemp, Warning, TEXT("BringAppToForeground: hWnd is null")); + } + } + else { + UE_LOG(LogTemp, Warning, TEXT("BringAppToForeground: NativeWindow invalid")); + } + } + else { + UE_LOG(LogTemp, Warning, TEXT("BringAppToForeground: MainWindow invalid")); + } #endif } @@ -1305,7 +1321,7 @@ void UBToolsBPLibrary::Minimize() void UBToolsBPLibrary::Maximize() { -//#if !WITH_EDITOR + //#if !WITH_EDITOR UGameEngine* gameEngine = Cast(GEngine); @@ -1320,7 +1336,7 @@ void UBToolsBPLibrary::Maximize() } } -//#endif // !WITH_EDITOR + //#endif // !WITH_EDITOR } bool UBToolsBPLibrary::IsFullscreen() @@ -1372,7 +1388,7 @@ bool UBToolsBPLibrary::IsWindowMaximized() void UBToolsBPLibrary::MoveWindowToMonitor(int monitorID) { -//#if !WITH_EDITOR + //#if !WITH_EDITOR if (GEngine && GEngine->GameViewport && GEngine->GameViewport->GetWindow().IsValid()) { @@ -1402,7 +1418,7 @@ void UBToolsBPLibrary::MoveWindowToMonitor(int monitorID) } } } -//#endif // !WITH_EDITOR + //#endif // !WITH_EDITOR } FVector2D UBToolsBPLibrary::GetWidgetScreenPosition(const UUserWidget* Widget) @@ -1417,7 +1433,7 @@ FVector2D UBToolsBPLibrary::GetWidgetScreenPosition(const UUserWidget* Widget) FGeometry Geometry = Widget->GetCachedGeometry(); FVector2D LocalCenter = Geometry.GetLocalSize() * 0.5f; FVector2D AbsoluteCenter = Geometry.LocalToAbsolute(LocalCenter); - + return AbsoluteCenter; } @@ -1425,10 +1441,10 @@ FWidgetLocationResult UBToolsBPLibrary::GetWorldLocationFromUIElement(const UUse { FWidgetLocationResult Result; if (!Widget) return Result; - + // Hole Widget Position auf Screen FVector2D WidgetScreenPosition = GetWidgetScreenPosition(Widget); - + UWorld* World = Widget->GetWorld(); if (!World) return Result; // Deproject von Screen zu World @@ -1437,7 +1453,7 @@ FWidgetLocationResult UBToolsBPLibrary::GetWorldLocationFromUIElement(const UUse UE_LOG(LogTemp, Warning, TEXT("WidgetScreenPosition BEFORE adjustments: %f, %f"), WidgetScreenPosition.X, WidgetScreenPosition.Y); UE_LOG(LogTemp, Warning, TEXT("Widget Center: %f, %f"), WidgetScreenPosition.X, WidgetScreenPosition.Y); - + // Update Postions so it is in center of the actual Widget //WidgetScreenPosition.X = WidgetScreenPosition.X - (Size.X/2);//Widget->GetRenderTransform().Scale.X; //WidgetScreenPosition.Y = WidgetScreenPosition.Y - (Size.Y/2); //- Widget->GetRenderTransform().Scale.Y; @@ -1445,7 +1461,7 @@ FWidgetLocationResult UBToolsBPLibrary::GetWorldLocationFromUIElement(const UUse UE_LOG(LogTemp, Warning, TEXT("WidgetScreenPosition AFTER adjustments: %f, %f"), WidgetScreenPosition.X, WidgetScreenPosition.Y); //UE_LOG(LogTemp, Warning, TEXT("Widget Absolute Size: %f, %f"), Widget->GetCachedGeometry().GetAbsoluteSize().X, Widget->GetCachedGeometry().GetAbsoluteSize().Y); UE_LOG(LogTemp, Warning, TEXT("Widget Absolute Size: %f, %f"), Size.X, Size.Y); - + FVector WorldLocation, WorldDirection; PC->DeprojectScreenPositionToWorld( WidgetScreenPosition.X, @@ -1463,16 +1479,16 @@ FVector UBToolsBPLibrary::GetWorldLocationFromUIElementWithRay(const UUserWidget { FVector Target = FVector::ZeroVector; if (!Widget) return Target; - + // Get widget screen position FVector2D WidgetScreenPosition = GetWidgetScreenPosition(Widget); - + UWorld* World = Widget->GetWorld(); if (!World) return Target; - + APlayerController* PC = World->GetFirstPlayerController(); if (!PC) return Target; - + // Deproject from screen to world FVector WorldLocation, WorldDirection; PC->DeprojectScreenPositionToWorld( @@ -1481,15 +1497,15 @@ FVector UBToolsBPLibrary::GetWorldLocationFromUIElementWithRay(const UUserWidget WorldLocation, WorldDirection ); - + // Now raycast forward from camera FVector TraceStart = WorldLocation; FVector TraceEnd = WorldLocation + (WorldDirection * TraceDistance); - + FHitResult HitResult; FCollisionQueryParams CollisionParams; CollisionParams.AddIgnoredActor(PC->GetPawn()); - + // Line trace to find world position if (World->LineTraceSingleByChannel( HitResult, @@ -1502,7 +1518,7 @@ FVector UBToolsBPLibrary::GetWorldLocationFromUIElementWithRay(const UUserWidget // Hit something - return hit location return HitResult.Location; } - + // No hit - return point at fixed distance return TraceStart + (WorldDirection * TraceDistance); } diff --git a/Unreal/Plugins/BTools/Source/BTools/Public/BToolsBPLibrary.h b/Unreal/Plugins/BTools/Source/BTools/Public/BToolsBPLibrary.h index 041414d..5ab2fcd 100644 --- a/Unreal/Plugins/BTools/Source/BTools/Public/BToolsBPLibrary.h +++ b/Unreal/Plugins/BTools/Source/BTools/Public/BToolsBPLibrary.h @@ -140,8 +140,9 @@ public: UFUNCTION(BlueprintCallable, Category = "BTools") static bool KillProcessByName(FString Name); + //Kill a process based on its path name. If UseAbolutePath is false all processes containing that FString dir will be killed. UFUNCTION(BlueprintCallable, Category = "BTools") - static int KillProcessesByPath(FString Dir); + static int KillProcessesByPath(FString Dir, bool UseAbsolutePath = true); UFUNCTION(BlueprintCallable, Category = "BTools") static bool KillProcessesByID(int ProcessID); diff --git a/Unreal/SPIE_Avatar.uproject b/Unreal/SPIE_Avatar.uproject index 8d560b6..3ee2d51 100644 --- a/Unreal/SPIE_Avatar.uproject +++ b/Unreal/SPIE_Avatar.uproject @@ -65,6 +65,10 @@ { "Name": "RigLogic", "Enabled": true + }, + { + "Name": "RawInput", + "Enabled": true } ], "TargetPlatforms": [ diff --git a/Unreal/SyncAvatarCore.bat b/Unreal/SyncAvatarCore.bat index 9704a9a..2697a40 100644 --- a/Unreal/SyncAvatarCore.bat +++ b/Unreal/SyncAvatarCore.bat @@ -49,7 +49,7 @@ if not exist "%SRC_ROOT%\" ( rem List of folders to mirror -set "FOLDERS=Plugins\AvatarCore_AI Plugins\AvatarCore_Manager Plugins\AvatarCore_MetaHuman Plugins\AvatarCore_STT Plugins\AvatarCore_TTS Plugins\BLogger Plugins\BSettings Plugins\BTools Plugins\RuntimeMetaHumanLipSync_5.6 Content\Project" +set "FOLDERS=Plugins\AvatarCore_AI Plugins\AvatarCore_Shared Plugins\AvatarCore_Manager Plugins\AvatarCore_MetaHuman Plugins\AvatarCore_STT Plugins\AvatarCore_TTS Plugins\BLogger Plugins\BSettings Plugins\BTools Plugins\RuntimeMetaHumanLipSync_5.6 Content\Project" set "ANY_FAIL=0" diff --git a/Unreal/SyncBackToAvatarCore.bat b/Unreal/SyncBackToAvatarCore.bat new file mode 100644 index 0000000..ab95656 --- /dev/null +++ b/Unreal/SyncBackToAvatarCore.bat @@ -0,0 +1,109 @@ +@echo off +setlocal EnableExtensions EnableDelayedExpansion + +rem ============================================================ +rem Mirror specific folders from A (AvatarBase) to B (this repo) +rem - Works relative to this .bat location +rem - Wipes extras in destination (/MIR) +rem - Logs removed extras (shown as *EXTRA File / *EXTRA Dir) +rem - Pauses before exit so console stays open +rem ============================================================ + +rem Destination root is the folder where this script lives +set "SRC_ROOT=%~dp0" +if "%SRC_ROOT:~-1%"=="\" set "SRC_ROOT=%SRC_ROOT:~0,-1%" + +rem Source root relative to this script +set "DST_ROOT=%SRC_ROOT%\..\..\AvatarBase\Unreal" + +rem Canonicalize paths +for %%I in ("%SRC_ROOT%") do set "SRC_ROOT=%%~fI" +for %%I in ("%DST_ROOT%") do set "DST_ROOT=%%~fI" + +echo ============================================================ +echo Robocopy Sync Script +echo ============================================================ +echo Source: %SRC_ROOT% +echo Destination: %DST_ROOT% +echo ============================================================ +echo. + +rem Guard: only intended for subprojects +if /I "%SRC_ROOT%"=="%DST_ROOT%" ( + echo ERROR: Source and destination are identical: + echo %SRC_ROOT% + echo This sync script is intended to be run from subprojects only. + echo. + pause + exit /b 1 +) + +if not exist "%SRC_ROOT%\" ( + echo ERROR: Source root does not exist: + echo %SRC_ROOT% + echo. + pause + exit /b 2 +) + +echo ============================================================ +echo Push back data from a project to avatar base? Are you sure? y/n +echo ============================================================ + +:PROMPT +SET /P AREYOUSURE=Are you sure (Y/[N])? +IF /I "%AREYOUSURE%" NEQ "Y" GOTO END + +rem List of folders to mirror +set "FOLDERS=Plugins\AvatarCore_AI Plugins\AvatarCore_Shared Plugins\AvatarCore_Manager Plugins\AvatarCore_MetaHuman Plugins\AvatarCore_STT Plugins\AvatarCore_TTS Plugins\BLogger Plugins\BSettings Plugins\BTools Plugins\RuntimeMetaHumanLipSync_5.6 Content\Project" + +set "ANY_FAIL=0" + +for %%F in (%FOLDERS%) do ( + set "SRC=%SRC_ROOT%\%%F" + set "DST=%DST_ROOT%\%%F" + + echo ------------------------------------------------------------ + echo Mirroring: %%F + echo FROM: !SRC! + echo TO: !DST! + echo ------------------------------------------------------------ + + if not exist "!SRC!\" ( + echo WARNING: Source folder missing, skipping: !SRC! + ) else ( + if not exist "!DST!\" ( + mkdir "!DST!" 2>nul + ) + + robocopy "!SRC!" "!DST!" /MIR /DCOPY:DAT /COPY:DAT /R:2 /W:1 /FFT /Z /NP /FP /TS /TEE + set "RC=!ERRORLEVEL!" + + echo Robocopy exit code: !RC! + + if !RC! GEQ 8 ( + echo ERROR: Robocopy reported failure for %%F ^(exit code !RC!^) + set "ANY_FAIL=1" + ) + ) + + echo. +) + +if "!ANY_FAIL!"=="1" ( + echo ============================================================ + echo Sync failed! + echo ============================================================ + pause + exit /b 8 +) + +echo ============================================================ +echo Sync finished. +echo ============================================================ + +:END +endlocal + +pause +exit /b 0 \ No newline at end of file