diff --git a/Unreal/Content/Project/AnimationTesting/BP_AnimationTesting_Manager.uasset b/Unreal/Content/Project/AnimationTesting/BP_AnimationTesting_Manager.uasset index 0843f52..5b8e4a4 100644 --- a/Unreal/Content/Project/AnimationTesting/BP_AnimationTesting_Manager.uasset +++ b/Unreal/Content/Project/AnimationTesting/BP_AnimationTesting_Manager.uasset @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:dbf1b9dbc5d89cb90030fd39f551d441cf82d073391640e4e32c3308b36365a1 -size 382167 +oid sha256:4219bcd34e2c85904f7b55dc209182d4322bde410e5a48ab8e60051975cc7230 +size 379827 diff --git a/Unreal/Content/Project/BP/BP_Project_Manager.uasset b/Unreal/Content/Project/BP/BP_Project_Manager.uasset index ca8f581..6841412 100644 --- a/Unreal/Content/Project/BP/BP_Project_Manager.uasset +++ b/Unreal/Content/Project/BP/BP_Project_Manager.uasset @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:cbd4979d1be26882f0eb2dc7a97209e9ef5260f371383f9c126ff38256d520ae -size 2481470 +oid sha256:dcd421a0daa75f4b54ae15bf923bf2cc36a80631783aa65d20b68c126aa72706 +size 2581241 diff --git a/Unreal/Content/Project/BP/EnumsAndStructs/S_ConfigSettings.uasset b/Unreal/Content/Project/BP/EnumsAndStructs/S_ConfigSettings.uasset deleted file mode 100644 index b8a3ed6..0000000 --- a/Unreal/Content/Project/BP/EnumsAndStructs/S_ConfigSettings.uasset +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:5a0e3faa1466076c0d45f1fd97676f3b7a3b402c4d68430dacf08594d79bc81d -size 69366 diff --git a/Unreal/Content/Project/BP/EnumsAndStructs/S_DEMO_Settings.uasset b/Unreal/Content/Project/BP/EnumsAndStructs/S_DEMO_Settings.uasset new file mode 100644 index 0000000..115cec7 --- /dev/null +++ b/Unreal/Content/Project/BP/EnumsAndStructs/S_DEMO_Settings.uasset @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:356080766cdceed9247b7471b959d5a582f349f2d821d26fdae06b5e54806dd7 +size 46007 diff --git a/Unreal/Content/Project/BP/EnumsAndStructs/S_ProjectBase_Settings.uasset b/Unreal/Content/Project/BP/EnumsAndStructs/S_ProjectBase_Settings.uasset new file mode 100644 index 0000000..70fd3d4 --- /dev/null +++ b/Unreal/Content/Project/BP/EnumsAndStructs/S_ProjectBase_Settings.uasset @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:212a2c64744f4348b8ee4d183715fed79aec677138cb90dd9a1b566d1ed1fd04 +size 20698 diff --git a/Unreal/Content/Project/Maps/M_Startup.umap b/Unreal/Content/Project/Maps/M_Startup.umap index 4da472d..4d640c8 100644 --- a/Unreal/Content/Project/Maps/M_Startup.umap +++ b/Unreal/Content/Project/Maps/M_Startup.umap @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:27527e1ec7a56ba04281237f9cb355bcc889abfed0158967f01db26d5d5ef7d5 -size 159627 +oid sha256:cdc3a71382e6633fb413afce31468696f929f20fe3d61c5be2887009c33a86ef +size 159495 diff --git a/Unreal/Content/Project/Widgets/W_DialogueBox.uasset b/Unreal/Content/Project/Widgets/W_DialogueBox.uasset index 3b3a696..defb63e 100644 --- a/Unreal/Content/Project/Widgets/W_DialogueBox.uasset +++ b/Unreal/Content/Project/Widgets/W_DialogueBox.uasset @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:0d9527f0cd47a5b0e09f2dd769529d4bc83ba0041f2110b442dff3039f0c4d85 -size 979065 +oid sha256:5c4b4e90d49269523f2b5974956ff685be11976c114325748ecf77759989750d +size 990921 diff --git a/Unreal/Content/Project/Widgets/W_Main.uasset b/Unreal/Content/Project/Widgets/W_Main.uasset index 91d530f..63487c2 100644 --- a/Unreal/Content/Project/Widgets/W_Main.uasset +++ b/Unreal/Content/Project/Widgets/W_Main.uasset @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:8c0a84a3f96a7aac1016c4544161e115340dd14ea08bb2232b6f8b1779a0e063 -size 403699 +oid sha256:83d6e226d9439d14a536fdd423139c325f533da51e92332b2cce2fc1805dc8cc +size 417365 diff --git a/Unreal/Content/SPIE/BP/S_SPIE_ConfigSettings.uasset b/Unreal/Content/SPIE/BP/S_SPIE_ConfigSettings.uasset index 748b063..327d15d 100644 --- a/Unreal/Content/SPIE/BP/S_SPIE_ConfigSettings.uasset +++ b/Unreal/Content/SPIE/BP/S_SPIE_ConfigSettings.uasset @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:bceba072a5ff554ef052a9bd0b7fe0e8c1b6fbb01f30155d2b8b36922d354e89 -size 87435 +oid sha256:18fe83b2a4c0d27ddd072b5d16c79e3541aa61444cb74e728db06fadb422f008 +size 69998 diff --git a/Unreal/Content/Schema/Spie_Config.schema.json b/Unreal/Content/Schema/Spie_Config.schema.json index 1cc0cf3..ff21900 100644 --- a/Unreal/Content/Schema/Spie_Config.schema.json +++ b/Unreal/Content/Schema/Spie_Config.schema.json @@ -1,10 +1,7 @@ { "Categories": [ - "SPIE", + "SPIESettings", "Project Setup", - "Debug", - "Volume", - "Engine Settings", "Avatar Core", "STT Settings", "STT", @@ -12,39 +9,12 @@ "TTS" ], "Variables": [ - { - "InitialMode": - { - "type": "integer", - "tooltip": "Which mode to load at startup", - "default": 0, - "category": "SPIE" - } - }, { "UseAvatarWithSafetyVest": { "type": "boolean", "default": true, - "category": "SPIE" - } - }, - { - "AvatarInstance": - { - "type": "string", - "tooltip": "Unique Name of this Avatar Application", - "default": "SPIE", - "category": "SPIE" - } - }, - { - "UseLogging": - { - "type": "boolean", - "tooltip": "Do you want to log all interaction?", - "default": true, - "category": "SPIE" + "category": "SPIESettings" } }, { @@ -54,16 +24,7 @@ "tooltip": "Showing in background logo", "default": "One SPIE. \r\nJust ask me.", "hotreload": true, - "category": "SPIE" - } - }, - { - "ButtonHintSpeech": - { - "type": "string", - "tooltip": "What to say, when user is allowed to speak?", - "default": "Drücke und halte beim Sprechen den Knopf vor dir, damit ich dich hören kann.", - "category": "SPIE" + "category": "SPIESettings" } }, { @@ -72,7 +33,7 @@ "type": "string", "tooltip": "What to say, when the Transcription was empty", "default": "Ich kann dich nicht hören: Drücke und halte den Knopf vor dir, um zu sprechen!", - "category": "SPIE" + "category": "SPIESettings" } }, { @@ -81,7 +42,7 @@ "type": "string", "tooltip": "What to say if the microphone does not seem to work", "default": "Das Mikrofon scheint nicht zu funktionieren - am besten suchst du dir Hilfe!", - "category": "SPIE" + "category": "SPIESettings" } }, { @@ -90,7 +51,7 @@ "type": "string", "tooltip": "Intro speech for the Innovation Day mode", "default": "Hallo und willkommen auf der One SPIE ! Ich bin ein virtueller Avatar mit dem du dich über unsere Hausmesse unterhalten kannst. By the way, you can talk in any language with me!", - "category": "SPIE" + "category": "SPIESettings" } }, { @@ -98,7 +59,7 @@ { "type": "float", "default": 20, - "category": "SPIE" + "category": "SPIESettings" } }, { @@ -106,7 +67,7 @@ { "type": "float", "default": 1, - "category": "SPIE" + "category": "SPIESettings" } }, { @@ -115,7 +76,7 @@ "type": "float", "default": 1, "hotreload": true, - "category": "SPIE" + "category": "SPIESettings" } }, { @@ -129,7 +90,7 @@ "Z": 0 }, "hotreload": true, - "category": "SPIE" + "category": "SPIESettings" } }, { @@ -137,7 +98,7 @@ { "type": "float", "default": 0.02, - "category": "SPIE" + "category": "SPIESettings" } }, { @@ -145,7 +106,7 @@ { "type": "float", "default": 10, - "category": "SPIE" + "category": "SPIESettings" } }, { @@ -153,223 +114,182 @@ { "type": "boolean", "default": false, - "category": "SPIE" - } - }, - { - "HideUI": - { - "type": "boolean", - "tooltip": "Hides the UI", - "default": false, - "hotreload": true, - "category": "Project Setup" - } - }, - { - "HideDialogueBoxAtStart": - { - "type": "boolean", - "tooltip": "If activated, the DialogueBox will hide after the first Button press to initialize the conversation. Can be show again by pressing \"H\"", - "default": false, - "hotreload": true, - "category": "Project Setup" - } - }, - { - "ConstrainAspectRatio": - { - "type": "boolean", - "tooltip": "If the camera should contrain to a vertical aspect ration. Can be used to enable a horizontal screen", - "default": false, - "hotreload": true, - "category": "Project Setup" + "category": "SPIESettings" } }, { - "CurrentLocation": + "BaseProjectSettings": { - "type": "string", - "tooltip": "Where is the user currently located?", - "default": "EUREF-Campus, 40472 Düsseldorf, Germany", + "type": "struct", + "fields": + { + "AvatarInstance": + { + "type": "string", + "tooltip": "Name of this Instance of the avatar application. Can be used for separation of logs or future purposes" + }, + "InitialMode": + { + "type": "integer", + "tooltip": "Which mode to load on startup" + }, + "UseLogging": + { + "type": "boolean" + }, + "ButtonHintSpeech": + { + "type": "string", + "tooltip": "Should the Avatar say something when QnA Mode starts" + }, + "HideUI": + { + "type": "boolean", + "tooltip": "Hides the UI" + }, + "HideDialogueBoxAtStart": + { + "type": "boolean", + "tooltip": "If activated, the DialogueBox will hide after the first Button press to initialize the conversation. Can be show again by pressing \"H\"" + }, + "ConstrainAspectRatio": + { + "type": "boolean", + "tooltip": "If the camera should contrain to a vertical aspect ration. Can be used to enable a horizontal screen" + }, + "AvatarVolume": + { + "type": "float" + }, + "VFXVolume": + { + "type": "float", + "tooltip": "Sound volume in general." + }, + "LoadAnimationTestmapOnStart": + { + "type": "boolean" + }, + "ConsoleCommands": + { + "type": "array", + "tooltip": "Console commands to always run", + "itemsType": "string" + }, + "CurrentLocation": + { + "type": "string", + "tooltip": "Where is the user located? To help the AI with its answers." + }, + "AvatarResetTimerAnimation": + { + "type": "float", + "tooltip": "For how many seconds of the end of the reset timer length should we show the circle animation?" + }, + "AvatarResetTimerLength": + { + "type": "float", + "tooltip": "How long to wait for a reset after avatar stopped talking. A value of 0 deactivates the timer" + }, + "UIReactionDistancePercentages": + { + "type": "array", + "tooltip": "3 Values From near to far: distances which indicate the alpha percentage of vector from player to current camera for UI reactions like clicking on buttons", + "itemsType": "float" + } + }, + "default": + { + "AvatarInstance": "Default Avatar", + "InitialMode": 0, + "UseLogging": true, + "ButtonHintSpeech": "Du kannst den Button drücken und halten, um mit mir zu sprechen.", + "HideUI": false, + "HideDialogueBoxAtStart": false, + "ConstrainAspectRatio": false, + "AvatarVolume": 1, + "VFXVolume": 1, + "LoadAnimationTestmapOnStart": false, + "ConsoleCommands": [], + "CurrentLocation": "", + "AvatarResetTimerAnimation": 15, + "AvatarResetTimerLength": 60, + "UIReactionDistancePercentages": [ 0.25, 0.5, 0.75 ] + }, "category": "Project Setup" } }, { - "DebugAI": - { - "type": "enum", - "tooltip": "Debugging mode for the AI Module", - "enum": [ - "Normal", - "DebugModule", - "DebugNoModule" - ], - "enumTypeName": "EAvatarCoreDebugModules", - "default": "Normal", - "category": "Debug" - } - }, - { - "DebugTTS": + "AvatarCoreSettings": { - "type": "enum", - "tooltip": "Debugging mode for the TTS Module", - "enum": [ - "Normal", - "DebugModule", - "DebugNoModule" - ], - "enumTypeName": "EAvatarCoreDebugModules", - "default": "Normal", - "category": "Debug" - } - }, - { - "DebugSTT": - { - "type": "enum", - "tooltip": "Debugging mode for the STT Module", - "enum": [ - "Normal", - "DebugModule", - "DebugNoModule" - ], - "enumTypeName": "EAvatarCoreDebugModules", - "default": "Normal", - "category": "Debug" - } - }, - { - "DebugAvatar": - { - "type": "boolean", - "tooltip": "Activated the debugging Mode for the Avatar", - "default": false, - "category": "Debug" - } - }, - { - "AvatarVolume": - { - "type": "float", - "tooltip": "Volume from 0-1 for the Avatar (Doesnt work with A2F)", - "default": 1, - "category": "Volume" - } - }, - { - "VFXVolume": - { - "type": "float", - "tooltip": "Volume from 0-1 for the VFX Sounds", - "default": 0.29999999999999999, - "hotreload": true, - "category": "Volume" - } - }, - { - "ConsoleCommands": - { - "type": "array", - "tooltip": "The one place to configure Console Commands", - "itemsType": "string", - "default": [ - "r.ScreenPercentage 100", - "t.MaxFPS 60", - "r.VSync 1", - "r.AntiAliasingMethod 2", - "r.RayTracing.Reflections.SamplesPerPixel 1", - "r.HairStrands.ComposeAfterTranslucency 0", - "r.HairStrands.DOFDepth 0", - "r.Lumen.ScreenProbeGather.DownsampleFactor 32", - "r.Lumen.Reflections.RadianceCache 1", - "r.Lumen.Reflections.MaxRoughnessToTraceClamp 0.3", - "r.Lumen.Reflections.AsyncCompute 1", - "r.RayTracing.Shadows 0", - "r.TemporalAASamples 16", - "r.RayTracing.Shadows.SamplesPerPixel 2" - ], - "hotreload": true, - "category": "Engine Settings" - } - }, - { - "LookAtEnabled": - { - "type": "boolean", - "tooltip": "Activated LookAt in Avatar AnimationSystem", - "default": true, - "category": "Avatar Core" - } - }, - { - "LookAtLocation": - { - "type": "vector3", - "tooltip": "Location for the LookAt", + "type": "struct", + "fields": + { + "DebugSTT": + { + "type": "enum", + "tooltip": "Deactivate or Debug STT Module", + "enum": [ + "Normal", + "DebugModule", + "DebugNoModule" + ], + "enumTypeName": "EAvatarCoreDebugModules" + }, + "DebugAI": + { + "type": "enum", + "tooltip": "Deactivate or Debug AI Module", + "enum": [ + "Normal", + "DebugModule", + "DebugNoModule" + ], + "enumTypeName": "EAvatarCoreDebugModules" + }, + "DebugTTS": + { + "type": "enum", + "tooltip": "Deactivate or Debug TTS Module", + "enum": [ + "Normal", + "DebugModule", + "DebugNoModule" + ], + "enumTypeName": "EAvatarCoreDebugModules" + }, + "LookAtEnabled": + { + "type": "boolean", + "tooltip": "IsLookAtEnabled" + }, + "LipSyncModel": + { + "type": "enum", + "enum": [ + "Original (Highest Quality)", + "Semi-Optimized (Balanced)", + "Highly Optimized (Fastest)" + ], + "enumTypeName": "ERealisticMetaHumanLipSyncModelType" + }, + "DebugAvatar": + { + "type": "boolean" + } + }, "default": { - "X": 0, - "Y": 560, - "Z": 110 + "DebugSTT": "Normal", + "DebugAI": "Normal", + "DebugTTS": "Normal", + "LookAtEnabled": true, + "LipSyncModel": "Original (Highest Quality)", + "DebugAvatar": false }, "category": "Avatar Core" } }, - { - "UseMCPServer": - { - "type": "boolean", - "tooltip": "Active MCP Server", - "default": false, - "category": "Avatar Core" - } - }, - { - "LipSyncModel": - { - "type": "enum", - "enum": [ - "Original (Highest Quality)", - "Semi-Optimized (Balanced)", - "Highly Optimized (Fastest)" - ], - "enumTypeName": "ERealisticMetaHumanLipSyncModelType", - "default": "Original (Highest Quality)", - "category": "Avatar Core" - } - }, - { - "AvatarResetTimerLength": - { - "type": "float", - "tooltip": "How long to wait for a reset after avatar stopped talking. A value of 0 deactivates the timer", - "default": 60, - "hotreload": true, - "category": "Avatar Core" - } - }, - { - "AvatarResetTimerAnimation": - { - "type": "float", - "tooltip": "For how many seconds of the end of the reset timer length should we show the circle animation?", - "default": 15, - "hotreload": true, - "category": "Avatar Core" - } - }, - { - "TalkByHoldTimerLength": - { - "type": "float", - "tooltip": "In seconds: If jumping to TalkToAvatar by pressing the button, how long to wait to skip the disclaimer if still holding the button?", - "default": 2, - "hotreload": true, - "category": "Avatar Core" - } - }, { "AzureSpeechService_API": { @@ -967,8 +887,9 @@ }, "WordReplacements": { - "type": "string", - "tooltip": "A map of words to replace in the text to fix pronauncation" + "type": "array", + "tooltip": "A map of words to replace in the text to fix pronauncation use separator \"|\". Left of it = word to preplace; right of it word to replace with.", + "itemsType": "string" }, "AudioNumChannels": { @@ -1076,6 +997,7 @@ "default": { "UseCacheSystem": true, + "WordReplacements": [], "AudioNumChannels": 1, "AudioSampleRate": 22050, "ResampleToSampleRate": -1, diff --git a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/.claude/settings.local.json b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/.claude/settings.local.json new file mode 100644 index 0000000..50c5d28 --- /dev/null +++ b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/.claude/settings.local.json @@ -0,0 +1,8 @@ +{ + "permissions": { + "allow": [ + "WebFetch(domain:openrouter.ai)", + "WebSearch" + ] + } +} diff --git a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/AIBaseManager.cpp b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/AIBaseManager.cpp index fcc076a..2a6a52d 100644 --- a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/AIBaseManager.cpp +++ b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/AIBaseManager.cpp @@ -167,8 +167,18 @@ void UAIBaseManager::SendResponse(FAIMessage Message, bool NotifyDelay) if (NotifyDelay) UAIBaseManager::StartDelayedAnswerTimer(); SendResponseChild(Message, NotifyDelay); - if(Message.Role == EAvatarCoreAIPromptRole::User || Message.Role == EAvatarCoreAIPromptRole::Tool) + if (Message.Role == EAvatarCoreAIPromptRole::User) AddMessageToArray(Message); + else if (Message.Role == EAvatarCoreAIPromptRole::Tool) + { + // ConfirmCommand may have already stored a placeholder with this Id. + // If so, CommandFinished updated it in-place — don't add a duplicate. + const bool bAlreadyStored = PreviousMessages.ContainsByPredicate([&](const FAIMessage& M) { + return M.Role == EAvatarCoreAIPromptRole::Tool && M.Id == Message.Id; + }); + if (!bAlreadyStored) + AddMessageToArray(Message); + } } void UAIBaseManager::RepeatText(FString TextToRepeat, bool DoRephrase) @@ -321,6 +331,7 @@ void UAIBaseManager::RunMCPCommand(FString CommandName, FString Payload, FString Cmd->SetWorldContext(WorldReferenceActor.Get()); ActiveCommands.Add(Cmd); + Cmd->OnCommandConfirmed.AddDynamic(this, &UAIBaseManager::CommandConfirmed); Cmd->OnCommandDone.AddDynamic(this, &UAIBaseManager::CommandFinished); Cmd->OnCommandFailed.AddDynamic(this, &UAIBaseManager::CommandFailed); @@ -363,6 +374,14 @@ FString UAIBaseManager::GetRoleAsString(EAvatarCoreAIPromptRole Role) } } +void UAIBaseManager::CommandConfirmed(const FAIMessage& Message) +{ + // Store the placeholder immediately so orphan detection doesn't strip the + // assistant tool_calls entry while the long-running task is in progress. + AddMessageToArray(Message); + BroadcastAILog(FString::Printf(TEXT("Command confirmed with placeholder: %s"), *Message.Message), true); +} + void UAIBaseManager::CommandFinished(const FAIMessage& Message) { ActiveCommands.RemoveAll([&Message](UMCPUnrealCommand* Cmd) @@ -377,6 +396,17 @@ void UAIBaseManager::CommandFinished(const FAIMessage& Message) else BroadcastAILog(TEXT("Command ran successfully."), true); + // If ConfirmCommand was called, a placeholder is already in history with this Id. + // Update it in-place so the AI receives the real result, not the interim message. + for (FAIMessage& Prev : PreviousMessages) + { + if (Prev.Role == EAvatarCoreAIPromptRole::Tool && Prev.Id == Message.Id) + { + Prev.Message = Message.Message; + break; + } + } + SendResponse(Message, false); } diff --git a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/MCP/MCPUnrealCommand.cpp b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/MCP/MCPUnrealCommand.cpp index 4bf0b58..0465a1d 100644 --- a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/MCP/MCPUnrealCommand.cpp +++ b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/MCP/MCPUnrealCommand.cpp @@ -52,6 +52,16 @@ void UMCPUnrealCommand::StartTimeout() } } +void UMCPUnrealCommand::ConfirmCommand(const FString& InProgressMessage) +{ + FAIMessage Msg; + Msg.Role = EAvatarCoreAIPromptRole::Tool; + Msg.Message = InProgressMessage; + Msg.Id = Id; + Msg.bTriggerResponse = false; + OnCommandConfirmed.Broadcast(Msg); +} + void UMCPUnrealCommand::FinishCommand(const FString& Payload, bool bTriggerResponse) { FAIMessage Msg; diff --git a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/OpenRouter/AvatarCoreAIOpenRouter.cpp b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/OpenRouter/AvatarCoreAIOpenRouter.cpp index e257e9d..e92a5b8 100644 --- a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/OpenRouter/AvatarCoreAIOpenRouter.cpp +++ b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/OpenRouter/AvatarCoreAIOpenRouter.cpp @@ -282,8 +282,16 @@ TArray> UAvatarCoreAIOpenRouter::BuildMessagesArray(FAIMe for (const FAIMessage& Msg : History) AppendMessage(Msg); - // Current message appended last — ensures it is always the newest entry - AppendMessage(CurrentMessage); + // If ConfirmCommand placed a placeholder in history and CommandFinished updated it, + // CurrentMessage (the final result) is already present — don't append it again. + const bool bCurrentAlreadyInHistory = + CurrentMessage.Role == EAvatarCoreAIPromptRole::Tool && !CurrentMessage.Id.IsEmpty() && + History.ContainsByPredicate([&](const FAIMessage& M) { + return M.Role == EAvatarCoreAIPromptRole::Tool && M.Id == CurrentMessage.Id; + }); + + if (!bCurrentAlreadyInHistory) + AppendMessage(CurrentMessage); return Messages; } diff --git a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/RealtimeAPI/AvatarCoreAIRealtime.cpp b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/RealtimeAPI/AvatarCoreAIRealtime.cpp index 01ebe5a..654ac67 100644 --- a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/RealtimeAPI/AvatarCoreAIRealtime.cpp +++ b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/RealtimeAPI/AvatarCoreAIRealtime.cpp @@ -590,7 +590,9 @@ void UAvatarCoreAIRealtime::OnWebSocketConnectionStringReceived(const FString& M { CurrentRequestID.Empty(); //Response is dead now CommandName = MessageJson->GetStringField(TEXT("name")); - RunMCPCommand(CommandName, MessageJson->GetStringField(TEXT("arguments"))); + FString CallId; + MessageJson->TryGetStringField(TEXT("call_id"), CallId); + RunMCPCommand(CommandName, MessageJson->GetStringField(TEXT("arguments")), CallId); } else { diff --git a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/Tools/OpenRouterResponder.cpp b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/Tools/OpenRouterResponder.cpp new file mode 100644 index 0000000..2baa963 --- /dev/null +++ b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/Tools/OpenRouterResponder.cpp @@ -0,0 +1,324 @@ +// Fill out your copyright notice in the Description page of Project Settings. + + +#include "Tools/OpenRouterResponder.h" + +#include "HttpModule.h" +#include "Interfaces/IHttpResponse.h" +#include "Dom/JsonObject.h" +#include "Serialization/JsonReader.h" +#include "Serialization/JsonSerializer.h" +#include "Serialization/JsonWriter.h" + +FString UOpenRouterResponder::StoredApiKey; +FString UOpenRouterResponder::StoredModel = TEXT("openai/gpt-4.1"); +FString UOpenRouterResponder::StoredSiteUrl; +FString UOpenRouterResponder::StoredSiteName; +TArray> UOpenRouterResponder::ActiveRequests; + +void UOpenRouterResponder::InitResponder(const FString& ApiKey, const FString& Model, const FString& SiteUrl, const FString& SiteName) +{ + StoredApiKey = ApiKey; + StoredModel = Model; + StoredSiteUrl = SiteUrl; + StoredSiteName = SiteName; +} + +UOpenRouterResponder* UOpenRouterResponder::CallOpenRouterResponse( + UObject* WorldContextObject, + const FString& Prompt, + const TArray& ServerTools, + const FString& Instructions, + const FString& OverrideModel, + int32 MaxOutputTokens, + ERouterReasoning Reasoning, + ERouterToolChoice ToolChoice) +{ + UOpenRouterResponder* Action = NewObject(); + Action->InputPrompt = Prompt; + Action->InputInstructions = Instructions; + Action->InputModel = OverrideModel.IsEmpty() ? StoredModel : OverrideModel; + Action->InputMaxOutputTokens = MaxOutputTokens; + Action->InputReasoning = Reasoning; + Action->InputServerTools = ServerTools; + Action->InputToolChoice = ToolChoice; + Action->RegisterWithGameInstance(WorldContextObject); + return Action; +} + +void UOpenRouterResponder::Activate() +{ + if (StoredApiKey.IsEmpty()) + { + UE_LOG(LogTemp, Error, TEXT("OpenRouterResponder: API key not set. Call InitResponder first.")); + Failed.Broadcast(TEXT("API key not set. Call InitResponder first.")); + return; + } + + RequestStartTime = FPlatformTime::Seconds(); + ActiveRequests.Add(this); + + // ── Build JSON request body ──────────────────────────────────────────── + + TSharedRef RootObject = MakeShared(); + RootObject->SetStringField(TEXT("model"), InputModel); + + // messages array: optional system + user + { + TArray> MessagesArray; + + if (!InputInstructions.IsEmpty()) + { + TSharedRef SystemMsg = MakeShared(); + SystemMsg->SetStringField(TEXT("role"), TEXT("system")); + SystemMsg->SetStringField(TEXT("content"), InputInstructions); + MessagesArray.Add(MakeShared(SystemMsg)); + } + + TSharedRef UserMsg = MakeShared(); + UserMsg->SetStringField(TEXT("role"), TEXT("user")); + UserMsg->SetStringField(TEXT("content"), InputPrompt); + MessagesArray.Add(MakeShared(UserMsg)); + + RootObject->SetArrayField(TEXT("messages"), MessagesArray); + } + + if (InputMaxOutputTokens > 0) + { + RootObject->SetNumberField(TEXT("max_tokens"), InputMaxOutputTokens); + } + + // Reasoning + if (InputReasoning != ERouterReasoning::None) + { + FString EffortStr; + switch (InputReasoning) + { + case ERouterReasoning::Minimal: EffortStr = TEXT("minimal"); break; + case ERouterReasoning::Low: EffortStr = TEXT("low"); break; + case ERouterReasoning::Medium: EffortStr = TEXT("medium"); break; + case ERouterReasoning::High: EffortStr = TEXT("high"); break; + case ERouterReasoning::XHigh: EffortStr = TEXT("xhigh"); break; + default: break; + } + + TSharedRef ReasoningObject = MakeShared(); + ReasoningObject->SetStringField(TEXT("effort"), EffortStr); + ReasoningObject->SetStringField(TEXT("summary"), TEXT("auto")); + RootObject->SetObjectField(TEXT("reasoning"), ReasoningObject); + } + + // Server tools array + if (InputServerTools.Num() > 0) + { + TArray> ToolsArray; + + for (const FRouterServerToolConfig& ToolConfig : InputServerTools) + { + TSharedRef ToolObject = MakeShared(); + + switch (ToolConfig.ToolType) + { + case ERouterServerTool::WebSearch: + ToolObject->SetStringField(TEXT("type"), TEXT("openrouter:web_search")); + if (ToolConfig.MaxResults > 0) + { + TSharedRef Params = MakeShared(); + Params->SetNumberField(TEXT("max_results"), ToolConfig.MaxResults); + ToolObject->SetObjectField(TEXT("parameters"), Params); + } + break; + case ERouterServerTool::DateTime: + ToolObject->SetStringField(TEXT("type"), TEXT("openrouter:datetime")); + break; + case ERouterServerTool::ImageGeneration: + ToolObject->SetStringField(TEXT("type"), TEXT("openrouter:image_generation")); + break; + case ERouterServerTool::WebFetch: + ToolObject->SetStringField(TEXT("type"), TEXT("openrouter:web_fetch")); + break; + } + + ToolsArray.Add(MakeShared(ToolObject)); + } + + RootObject->SetArrayField(TEXT("tools"), ToolsArray); + + // Only send tool_choice when the caller specified Auto or Required + if (InputToolChoice == ERouterToolChoice::Auto) + { + RootObject->SetStringField(TEXT("tool_choice"), TEXT("auto")); + } + else if (InputToolChoice == ERouterToolChoice::Required) + { + RootObject->SetStringField(TEXT("tool_choice"), TEXT("required")); + } + } + + // Serialize to string + FString Body; + TSharedRef> Writer = TJsonWriterFactory<>::Create(&Body); + FJsonSerializer::Serialize(RootObject, Writer); + + // ── Fire HTTP request ────────────────────────────────────────────────── + + TSharedRef HttpRequest = FHttpModule::Get().CreateRequest(); + HttpRequest->SetURL(TEXT("https://openrouter.ai/api/v1/chat/completions")); + HttpRequest->SetVerb(TEXT("POST")); + HttpRequest->SetHeader(TEXT("Content-Type"), TEXT("application/json")); + HttpRequest->SetHeader(TEXT("Authorization"), FString::Printf(TEXT("Bearer %s"), *StoredApiKey)); + + if (!StoredSiteUrl.IsEmpty()) + { + HttpRequest->SetHeader(TEXT("HTTP-Referer"), StoredSiteUrl); + } + if (!StoredSiteName.IsEmpty()) + { + HttpRequest->SetHeader(TEXT("X-Title"), StoredSiteName); + } + + HttpRequest->SetContentAsString(Body); + HttpRequest->OnProcessRequestComplete().BindUObject(this, &UOpenRouterResponder::HandleHttpResponse); + + if (!HttpRequest->ProcessRequest()) + { + UE_LOG(LogTemp, Error, TEXT("OpenRouterResponder: Failed to start HTTP request.")); + Failed.Broadcast(TEXT("Failed to start HTTP request.")); + } +} + +void UOpenRouterResponder::ClearAllResponses() +{ + TArray> RequestsCopy = ActiveRequests; + ActiveRequests.Empty(); + + for (const TWeakObjectPtr& Weak : RequestsCopy) + { + if (UOpenRouterResponder* Action = Weak.Get()) + { + Action->SetReadyToDestroy(); + } + } +} + +void UOpenRouterResponder::HandleHttpResponse(FHttpRequestPtr Request, FHttpResponsePtr Response, bool bWasSuccessful) +{ + ActiveRequests.RemoveAll([this](const TWeakObjectPtr& Weak) + { + return !Weak.IsValid() || Weak.Get() == this; + }); + + const float ElapsedTime = static_cast(FPlatformTime::Seconds() - RequestStartTime); + + if (!bWasSuccessful || !Response.IsValid()) + { + UE_LOG(LogTemp, Error, TEXT("OpenRouterResponder: HTTP request failed.")); + Failed.Broadcast(TEXT("HTTP request failed.")); + return; + } + + const int32 Code = Response->GetResponseCode(); + const FString Body = Response->GetContentAsString(); + + if (Code < 200 || Code >= 300) + { + UE_LOG(LogTemp, Error, TEXT("OpenRouterResponder: HTTP %d — %s"), Code, *Body); + Failed.Broadcast(FString::Printf(TEXT("HTTP %d — %s"), Code, *Body)); + return; + } + + // Parse root JSON + TSharedPtr RootObject; + TSharedRef> Reader = TJsonReaderFactory<>::Create(Body); + if (!FJsonSerializer::Deserialize(Reader, RootObject) || !RootObject.IsValid()) + { + UE_LOG(LogTemp, Error, TEXT("OpenRouterResponder: Failed to parse response JSON.")); + Failed.Broadcast(TEXT("Failed to parse response JSON.")); + return; + } + + // Check for API-level error + if (RootObject->HasField(TEXT("error"))) + { + const TSharedPtr* ErrorObject; + if (RootObject->TryGetObjectField(TEXT("error"), ErrorObject)) + { + FString ErrorMsg = (*ErrorObject)->GetStringField(TEXT("message")); + UE_LOG(LogTemp, Error, TEXT("OpenRouterResponder: API error — %s"), *ErrorMsg); + Failed.Broadcast(ErrorMsg); + return; + } + } + + // Navigate choices[0].message + const TArray>* ChoicesArray; + if (!RootObject->TryGetArrayField(TEXT("choices"), ChoicesArray) || ChoicesArray->Num() == 0) + { + UE_LOG(LogTemp, Error, TEXT("OpenRouterResponder: No choices in response.")); + Failed.Broadcast(TEXT("No choices in response.")); + return; + } + + const TSharedPtr& FirstChoice = (*ChoicesArray)[0]->AsObject(); + if (!FirstChoice.IsValid()) + { + UE_LOG(LogTemp, Error, TEXT("OpenRouterResponder: Invalid choice object.")); + Failed.Broadcast(TEXT("Invalid choice object.")); + return; + } + + const TSharedPtr* MessageObjectPtr; + if (!FirstChoice->TryGetObjectField(TEXT("message"), MessageObjectPtr) || !(*MessageObjectPtr).IsValid()) + { + UE_LOG(LogTemp, Error, TEXT("OpenRouterResponder: No message in choice.")); + Failed.Broadcast(TEXT("No message in choice.")); + return; + } + + const TSharedPtr& MessageObject = *MessageObjectPtr; + + FRouterResult Result; + Result.ExecutionTime = ElapsedTime; + + // content can be a plain string or, for multimodal models, an array — handle both + { + FString ContentStr; + if (MessageObject->TryGetStringField(TEXT("content"), ContentStr)) + { + Result.Content = ContentStr; + } + else + { + const TArray>* ContentArray; + if (MessageObject->TryGetArrayField(TEXT("content"), ContentArray)) + { + for (const TSharedPtr& Item : *ContentArray) + { + const TSharedPtr& ItemObj = Item->AsObject(); + if (!ItemObj.IsValid()) continue; + + FString ItemType; + FString ItemText; + if (ItemObj->TryGetStringField(TEXT("type"), ItemType) && ItemType == TEXT("text") + && ItemObj->TryGetStringField(TEXT("text"), ItemText)) + { + Result.Content += ItemText; + } + } + } + } + } + + // reasoning is optional — present only when the model emits chain-of-thought + MessageObject->TryGetStringField(TEXT("reasoning"), Result.Reasoning); + + if (!Result.Content.IsEmpty() || !Result.Reasoning.IsEmpty()) + { + Success.Broadcast(Result); + } + else + { + UE_LOG(LogTemp, Warning, TEXT("OpenRouterResponder: Response contained no content or reasoning.")); + Failed.Broadcast(TEXT("Response contained no content or reasoning.")); + } +} diff --git a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/AIBaseConfig.h b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/AIBaseConfig.h index a94c06b..5d19223 100644 --- a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/AIBaseConfig.h +++ b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/AIBaseConfig.h @@ -35,7 +35,7 @@ struct FGlobalAISettings // Boot up a FastMCP Server on Startup (On close keep open in Editor Mode, kill in shipping) UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|MCP", meta = (ExposeOnSpawn = "true")) - bool bUseMCPServer = true; + bool bUseMCPServer = false; // Does the AI model generate Audio Chunks that can be forwarded to the TTS Manager? UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Settings", meta = (ExposeOnSpawn = "true")) diff --git a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/AIBaseManager.h b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/AIBaseManager.h index c4df633..9293bd3 100644 --- a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/AIBaseManager.h +++ b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/AIBaseManager.h @@ -227,6 +227,11 @@ public: protected: + /** Bound to UMCPUnrealCommand::OnCommandConfirmed — stores a placeholder tool result + * immediately so the assistant tool_calls entry is never seen as orphaned during long tasks. */ + UFUNCTION() + void CommandConfirmed(const FAIMessage& Message); + /** Bound to UMCPUnrealCommand::OnCommandDone — message already has Role=Tool and Id set. */ UFUNCTION() void CommandFinished(const FAIMessage& Message); diff --git a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/MCP/MCPUnrealCommand.h b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/MCP/MCPUnrealCommand.h index d204e60..a2480d8 100644 --- a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/MCP/MCPUnrealCommand.h +++ b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/MCP/MCPUnrealCommand.h @@ -7,6 +7,7 @@ DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FOnAICommandDone, const FAIMessage&, Message); DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FOnAICommandFailed, const FAIMessage&, Message); +DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FOnAICommandConfirmed, const FAIMessage&, Message); /** * Base class for MCP/AI commands. @@ -44,6 +45,12 @@ public: UPROPERTY(BlueprintAssignable, Category = "Command") FOnAICommandFailed OnCommandFailed; + // Intermediate confirmation — call this at the start of a long-running task to + // place a placeholder tool result in history immediately. FinishCommand/FailCommand + // will update the placeholder with the real result when the task completes. + UPROPERTY(BlueprintAssignable, Category = "Command") + FOnAICommandConfirmed OnCommandConfirmed; + /** * Execute the command using the provided world reference and a JSON payload from the AI model. * All world/actor operations should use this reference. The payload contains arguments or context as JSON. @@ -74,6 +81,11 @@ public: UFUNCTION(BlueprintCallable, Category = "Command") FString GetCommandOutputScheme(); + /** Call at the start of a long-running task to place a placeholder in message history. + * FinishCommand/FailCommand will replace it with the real result when done. */ + UFUNCTION(BlueprintCallable, Category = "Command") + void ConfirmCommand(const FString& InProgressMessage); + /** Call this when the command is finished successfully. Set bTriggerResponse=false to suppress the AI follow-up. */ UFUNCTION(BlueprintCallable, Category = "Command") void FinishCommand(const FString& Payload, bool bTriggerResponse = true); diff --git a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/Tools/OpenRouterResponder.h b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/Tools/OpenRouterResponder.h new file mode 100644 index 0000000..2937c7f --- /dev/null +++ b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/Tools/OpenRouterResponder.h @@ -0,0 +1,136 @@ +// Fill out your copyright notice in the Description page of Project Settings. + +#pragma once + +#include "CoreMinimal.h" +#include "Kismet/BlueprintAsyncActionBase.h" +#include "Http.h" +#include "OpenRouterResponder.generated.h" + +// ── Enums ────────────────────────────────────────────────────────────────── + +UENUM(BlueprintType) +enum class ERouterToolChoice : uint8 +{ + None UMETA(DisplayName = "None (omit)"), + Auto UMETA(DisplayName = "Auto"), + Required UMETA(DisplayName = "Required") +}; + +UENUM(BlueprintType) +enum class ERouterReasoning : uint8 +{ + None UMETA(DisplayName = "None"), + Minimal UMETA(DisplayName = "Minimal"), + Low UMETA(DisplayName = "Low"), + Medium UMETA(DisplayName = "Medium"), + High UMETA(DisplayName = "High"), + XHigh UMETA(DisplayName = "XHigh") +}; + +UENUM(BlueprintType) +enum class ERouterServerTool : uint8 +{ + WebSearch UMETA(DisplayName = "Web Search"), + DateTime UMETA(DisplayName = "Date/Time"), + ImageGeneration UMETA(DisplayName = "Image Generation"), + WebFetch UMETA(DisplayName = "Web Fetch") +}; + +// ── Structs ──────────────────────────────────────────────────────────────── + +USTRUCT(BlueprintType) +struct FRouterServerToolConfig +{ + GENERATED_BODY() + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Router") + ERouterServerTool ToolType = ERouterServerTool::WebSearch; + + /** Maximum results returned. Only used when ToolType is WebSearch. */ + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Router") + int32 MaxResults = 3; +}; + +USTRUCT(BlueprintType) +struct FRouterResult +{ + GENERATED_BODY() + + /** Main text content from choices[0].message.content */ + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Router") + FString Content; + + /** Chain-of-thought from choices[0].message.reasoning. Empty if the model does not return it. */ + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Router") + FString Reasoning; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Router") + float ExecutionTime = 0.f; +}; + +// ── Delegates ────────────────────────────────────────────────────────────── + +DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FOnRouterSuccess, const FRouterResult&, Result); +DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FOnRouterFailed, const FString&, ErrorMessage); + +// ── Class ────────────────────────────────────────────────────────────────── + +UCLASS() +class AVATARCORE_AI_API UOpenRouterResponder : public UBlueprintAsyncActionBase +{ + GENERATED_BODY() + +public: + /** Store API key, default model, and optional attribution headers sent to OpenRouter. */ + UFUNCTION(BlueprintCallable, Category = "AvatarCoreAI|Router", meta = (DisplayName = "Init OpenRouter Responder")) + static void InitResponder( + const FString& ApiKey, + const FString& Model = TEXT("openai/gpt-4.1"), + const FString& SiteUrl = TEXT(""), + const FString& SiteName = TEXT("")); + + /** Send a one-shot request to the OpenRouter Chat Completions API. */ + UFUNCTION(BlueprintCallable, Category = "AvatarCoreAI|Router", + meta = (BlueprintInternalUseOnly = "true", DisplayName = "Call OpenRouter Response", WorldContext = "WorldContextObject", + AutoCreateRefTerm = "Instructions,OverrideModel,ServerTools")) + static UOpenRouterResponder* CallOpenRouterResponse( + UObject* WorldContextObject, + const FString& Prompt, + const TArray& ServerTools, + const FString& Instructions, + const FString& OverrideModel, + int32 MaxOutputTokens = 0, + ERouterReasoning Reasoning = ERouterReasoning::None, + ERouterToolChoice ToolChoice = ERouterToolChoice::Auto); + + /** Cancel all active OpenRouter requests. */ + UFUNCTION(BlueprintCallable, Category = "AvatarCoreAI|Router", meta = (DisplayName = "Clear All Router Responses")) + static void ClearAllResponses(); + + UPROPERTY(BlueprintAssignable) + FOnRouterSuccess Success; + + UPROPERTY(BlueprintAssignable) + FOnRouterFailed Failed; + + virtual void Activate() override; + +private: + void HandleHttpResponse(FHttpRequestPtr Request, FHttpResponsePtr Response, bool bWasSuccessful); + + FString InputPrompt; + FString InputInstructions; + FString InputModel; + int32 InputMaxOutputTokens = 0; + ERouterReasoning InputReasoning = ERouterReasoning::None; + TArray InputServerTools; + ERouterToolChoice InputToolChoice = ERouterToolChoice::Auto; + double RequestStartTime = 0.0; + + static FString StoredApiKey; + static FString StoredModel; + static FString StoredSiteUrl; + static FString StoredSiteName; + static TArray> ActiveRequests; +}; diff --git a/Unreal/Plugins/AvatarCore_Manager/Content/AvatarCoreManager.uasset b/Unreal/Plugins/AvatarCore_Manager/Content/AvatarCoreManager.uasset index 5da1910..5e84c0e 100644 --- a/Unreal/Plugins/AvatarCore_Manager/Content/AvatarCoreManager.uasset +++ b/Unreal/Plugins/AvatarCore_Manager/Content/AvatarCoreManager.uasset @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:9379803e1aa60bc5952e611cebb9947689372c8bf255cdc42166c79cb7dc34a7 -size 2129983 +oid sha256:1968895b296e8afec8db3b6546018f0c99ad8620470d7b913ea7d2dc5129891d +size 2130952 diff --git a/Unreal/Plugins/AvatarCore_Manager/Content/S_AvatarCoreSettings.uasset b/Unreal/Plugins/AvatarCore_Manager/Content/S_AvatarCoreSettings.uasset index a1a63f0..ffbbdd7 100644 --- a/Unreal/Plugins/AvatarCore_Manager/Content/S_AvatarCoreSettings.uasset +++ b/Unreal/Plugins/AvatarCore_Manager/Content/S_AvatarCoreSettings.uasset @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:cbb52cbe768a4aecae2b1b62c9a8d05137cfa250045efa16b2403d55b4a47abc -size 6456 +oid sha256:66c9b22d77e39cf18c07f8218243f64661edc8367f2dd930e1531d06e5f9d909 +size 9885 diff --git a/Unreal/Plugins/AvatarCore_Manager/Content/StateManagement/BP_StateManager.uasset b/Unreal/Plugins/AvatarCore_Manager/Content/StateManagement/BP_StateManager.uasset index 4ee06f8..aebb9aa 100644 --- a/Unreal/Plugins/AvatarCore_Manager/Content/StateManagement/BP_StateManager.uasset +++ b/Unreal/Plugins/AvatarCore_Manager/Content/StateManagement/BP_StateManager.uasset @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:35f47b127b32456b3df115db0bb4bb5a248bfb02da7952cd931553ff6931ee82 -size 460348 +oid sha256:2f8d000da145cda9339965cdd2cd78005ff245c1fc45e9428ef6f1b1ac2647e8 +size 451020 diff --git a/Unreal/Plugins/AvatarCore_Manager/Content/StateManagement/States/BP_QnA_State.uasset b/Unreal/Plugins/AvatarCore_Manager/Content/StateManagement/States/BP_QnA_State.uasset index d6149cd..f4da9b8 100644 --- a/Unreal/Plugins/AvatarCore_Manager/Content/StateManagement/States/BP_QnA_State.uasset +++ b/Unreal/Plugins/AvatarCore_Manager/Content/StateManagement/States/BP_QnA_State.uasset @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:8f2206168cb1c65d310b7501bad2bfb65e99ed31e7d99df2380c2d352d3a76da -size 198359 +oid sha256:b304180aac6b22b077707d80ef93b7f7c2cd16cb5186d55be3086ff2045f185a +size 221012 diff --git a/Unreal/Plugins/AvatarCore_Manager/Content/UnrealCommands/AICommand_CurrentLocation.uasset b/Unreal/Plugins/AvatarCore_Manager/Content/UnrealCommands/AICommand_CurrentLocation.uasset index 8783e78..cfcc03a 100644 --- a/Unreal/Plugins/AvatarCore_Manager/Content/UnrealCommands/AICommand_CurrentLocation.uasset +++ b/Unreal/Plugins/AvatarCore_Manager/Content/UnrealCommands/AICommand_CurrentLocation.uasset @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:b1a33d50214a3cfe8a86331ffe371170db7ba6a5ad00c600b85efa7c956ba84f -size 35244 +oid sha256:69fecb1290b5ea7f08b52a1f1b1334735e4b4adc13efaa49fc229301cfb09c5b +size 50670 diff --git a/Unreal/Plugins/AvatarCore_Manager/Content/UnrealCommands/OpenAI_Websearch_Command.uasset b/Unreal/Plugins/AvatarCore_Manager/Content/UnrealCommands/OpenAI_Websearch_Command.uasset index 8b3b33a..c53cfef 100644 --- a/Unreal/Plugins/AvatarCore_Manager/Content/UnrealCommands/OpenAI_Websearch_Command.uasset +++ b/Unreal/Plugins/AvatarCore_Manager/Content/UnrealCommands/OpenAI_Websearch_Command.uasset @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:8656f137eadd663be7a1eabda17db6d64230ca1f540cf15e9d003334133a1f9c -size 79890 +oid sha256:d837244f4f32007621395b1369795863890f895da4099ab39eb21c2e79091d13 +size 2641 diff --git a/Unreal/Plugins/AvatarCore_Manager/Content/UnrealCommands/WebResearch_Command.uasset b/Unreal/Plugins/AvatarCore_Manager/Content/UnrealCommands/WebResearch_Command.uasset new file mode 100644 index 0000000..a4bccdc --- /dev/null +++ b/Unreal/Plugins/AvatarCore_Manager/Content/UnrealCommands/WebResearch_Command.uasset @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:af3e64662aa6ceb94086af6daa79ff93ddfa9e26f235b3d5bcda36646a34e829 +size 115900 diff --git a/Unreal/Plugins/AvatarCore_MetaHuman/Content/Animation/AnimationConfigs/AnimationConfig_Base.uasset b/Unreal/Plugins/AvatarCore_MetaHuman/Content/Animation/AnimationConfigs/AnimationConfig_Base.uasset index b2c75ca..0f8fd3c 100644 --- a/Unreal/Plugins/AvatarCore_MetaHuman/Content/Animation/AnimationConfigs/AnimationConfig_Base.uasset +++ b/Unreal/Plugins/AvatarCore_MetaHuman/Content/Animation/AnimationConfigs/AnimationConfig_Base.uasset @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:bd5226ee2db8e15bf08ce02380f3dd85dc8baf17485bf562e7fbaa97acf61362 -size 38331 +oid sha256:5e13752724da927286cfa2d2b71381596f3442789e3d6ad830040573c86fd0f3 +size 37792 diff --git a/Unreal/Plugins/AvatarCore_MetaHuman/Content/BP/MetaHuman/BaseAvatar.uasset b/Unreal/Plugins/AvatarCore_MetaHuman/Content/BP/MetaHuman/BaseAvatar.uasset index 5f2152c..f4ad5b1 100644 --- a/Unreal/Plugins/AvatarCore_MetaHuman/Content/BP/MetaHuman/BaseAvatar.uasset +++ b/Unreal/Plugins/AvatarCore_MetaHuman/Content/BP/MetaHuman/BaseAvatar.uasset @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:22d7cc09dd8c7265ead02841968633ea72ec7c5e7d0ac82f180d9da451760850 -size 2490354 +oid sha256:1c2231e572a36ea32e73c9489ce9adbc986c080ed6fcec9158b71641829a362d +size 2500595 diff --git a/Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Private/Cartesia/CartesiaTTSManager.cpp b/Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Private/Cartesia/CartesiaTTSManager.cpp index 6753742..cd3f2cd 100644 --- a/Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Private/Cartesia/CartesiaTTSManager.cpp +++ b/Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Private/Cartesia/CartesiaTTSManager.cpp @@ -23,25 +23,6 @@ namespace FJsonSerializer::Serialize(Obj.ToSharedRef(), Writer); return Out; } - - static const FString TTSLanguageStrings[] = { - TEXT("en"), TEXT("fr"), TEXT("de"), TEXT("es"), TEXT("pt"), TEXT("zh"), TEXT("ja"), - TEXT("hi"), TEXT("it"), TEXT("ko"), TEXT("nl"), TEXT("pl"), TEXT("ru"), TEXT("sv"), - TEXT("tr"), TEXT("tl"), TEXT("bg"), TEXT("ro"), TEXT("ar"), TEXT("cs"), TEXT("el"), - TEXT("fi"), TEXT("hr"), TEXT("ms"), TEXT("sk"), TEXT("da"), TEXT("ta"), TEXT("uk"), - TEXT("hu"), TEXT("no"), TEXT("vi"), TEXT("bn"), TEXT("th"), TEXT("he"), TEXT("ka"), - TEXT("id"), TEXT("te"), TEXT("gu"), TEXT("kn"), TEXT("ml"), TEXT("mr"), TEXT("pa") - }; - - static FString TTSLanguageToString(ETTSLanguage Language) - { - const int32 Index = static_cast(Language); - if (Index >= 0 && Index < UE_ARRAY_COUNT(TTSLanguageStrings)) - { - return TTSLanguageStrings[Index]; - } - return TEXT("de"); - } } FString UCartesiaTTSManager::BuildWebSocketUrl() const @@ -315,7 +296,7 @@ void UCartesiaTTSManager::StartStreamingGeneration(int32 TaskID, const FString& { if (!WeakThis.IsValid()) return; UCartesiaTTSManager* Self = WeakThis.Get(); - Self->TTSLog(FString::Printf(TEXT("[Cartesia][%d] <- %s"), TaskID, *Message)); + //Self->TTSLog(FString::Printf(TEXT("[Cartesia][%d] <- %s"), TaskID, *Message)); TSharedPtr Obj; TSharedRef> Reader = TJsonReaderFactory<>::Create(Message); diff --git a/Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Private/TTSManagerBase.cpp b/Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Private/TTSManagerBase.cpp index d3727f1..f285228 100644 --- a/Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Private/TTSManagerBase.cpp +++ b/Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Private/TTSManagerBase.cpp @@ -205,18 +205,24 @@ FString UTTSManagerBase::ProcessTextForGeneration(const FString& Input) // Apply word/phrase replacements from config (case-insensitive) if (TTSConfig && TTSConfig->GlobalTTSSettings.WordReplacements.Num() > 0) { - for (const TPair& Pair : TTSConfig->GlobalTTSSettings.WordReplacements) + for (const FString& Entry : TTSConfig->GlobalTTSSettings.WordReplacements) { - const FString& From = Pair.Key; - const FString& To = Pair.Value; - if (!From.IsEmpty()) + FString From; + FString To; + + // Split "from|to" + if (Entry.Split(TEXT("|"), &From, &To)) { - Result.ReplaceInline(*From, *To, ESearchCase::CaseSensitive); + From = From.TrimStartAndEnd(); + To = To.TrimStartAndEnd(); + + if (!From.IsEmpty()) + { + Result.ReplaceInline(*From, *To, ESearchCase::IgnoreCase); + } } } } - - return Result; } diff --git a/Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Public/Cartesia/TTSCartesiaConfig.h b/Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Public/Cartesia/TTSCartesiaConfig.h index 2545678..b785391 100644 --- a/Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Public/Cartesia/TTSCartesiaConfig.h +++ b/Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Public/Cartesia/TTSCartesiaConfig.h @@ -18,10 +18,10 @@ struct FCartesiaTTSSettings FString CartesiaVoiceId = TEXT("e00dd3df-19e7-4cd4-827a-7ff6687b6954"); UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreTTS|Cartesia", meta = (ExposeOnSpawn = "true")) - FString CartesiaModelId = TEXT("sonic-3"); + FString CartesiaModelId = TEXT("sonic-3.5"); UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreTTS|Cartesia", meta = (ExposeOnSpawn = "true")) - FString CartesiaLanguage = TEXT("en"); + FString CartesiaLanguage = TEXT("de"); UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreTTS|Cartesia", meta = (ExposeOnSpawn = "true")) bool StreamInputText = true; diff --git a/Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Public/TTSBaseConfig.h b/Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Public/TTSBaseConfig.h index b02803c..a058fd9 100644 --- a/Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Public/TTSBaseConfig.h +++ b/Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Public/TTSBaseConfig.h @@ -69,6 +69,21 @@ enum class ETTSLanguage : uint8 pa UMETA(DisplayName = "Punjabi"), }; +// Returns the ISO 639-1 code string for the given language enum (e.g. ETTSLanguage::de → "de"). +// Derived from the enum value name via UE reflection, so it stays in sync automatically. +static FORCEINLINE FString TTSLanguageToString(ETTSLanguage Language) +{ + FString Name = StaticEnum()->GetNameStringByValue(static_cast(Language)); + // StaticEnum returns the short name (no "ETTSLanguage::" prefix in UE5 GetNameStringByValue) + // but strip the prefix defensively in case the runtime returns the qualified form. + int32 ScopePos; + if (Name.FindLastChar(TEXT(':'), ScopePos)) + { + Name = Name.RightChop(ScopePos + 1); + } + return Name.IsEmpty() ? TEXT("en") : Name; +} + USTRUCT(BlueprintType) struct FGlobalTTSSettings { @@ -78,9 +93,9 @@ struct FGlobalTTSSettings UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreTTS|Base", meta = (ExposeOnSpawn = "true")) bool UseCacheSystem = true; - //A map of words to replace in the text to fix pronauncation + //A map of words to replace in the text to fix pronauncation use separator "|". Left of it = word to preplace; right of it word to replace with. UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreTTS|Base", meta = (ExposeOnSpawn = "true")) - TMap WordReplacements; + TArray WordReplacements = {"b.ReX|biRäx", "AI|EyEi"}; //Number of Audio Channels - Only tested with 1 but who needs more anyway? UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreTTS|Base", meta = (ExposeOnSpawn = "true"))