Browse Source

Matched with Base

master
Tillman Staffen 1 week ago
parent
commit
06dff0030b
  1. 1
      Unreal/Config/DefaultInput.ini
  2. BIN
      Unreal/Content/Project/BP/BP_Project_Manager.uasset
  3. BIN
      Unreal/Content/Project/BP/EnumsAndStructs/S_DEMO_Settings.uasset
  4. BIN
      Unreal/Content/Project/Widgets/ChildWidgets/W_ToolcallEntry.uasset
  5. BIN
      Unreal/Content/Project/Widgets/Debug/W_DebugStateWidget.uasset
  6. BIN
      Unreal/Content/Project/Widgets/W_DialogueBox.uasset
  7. BIN
      Unreal/Content/Project/Widgets/W_Toolcall.uasset
  8. BIN
      Unreal/Content/Project/Widgets/W_Toolcall_Simple.uasset
  9. BIN
      Unreal/Content/SPIE/BP/BP_SPIE_Manager_Child.uasset
  10. BIN
      Unreal/Content/SPIE/BP/S_SPIE_ConfigSettings.uasset
  11. 132
      Unreal/Content/Schema/Spie_Config.schema.json
  12. 6
      Unreal/Plugins/AvatarCore_AI/AvatarCore_AI.uplugin
  13. 8
      Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/AvatarCore_AI.Build.cs
  14. 90
      Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/AIBaseManager.cpp
  15. 38
      Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/MCP/FastMCP/FastMCPManager.cpp
  16. 28
      Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/MCP/MCPUnrealCommand.cpp
  17. 4
      Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/RealtimeAPI/AvatarCoreAIRealtime.cpp
  18. 178
      Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/Tools/OpenAiModerator.cpp
  19. 7
      Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/Tools/OpenAiResponder.cpp
  20. 7
      Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/Tools/OpenRouterResponder.cpp
  21. 4
      Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/AIBaseConfig.h
  22. 43
      Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/AIBaseManager.h
  23. 30
      Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/MCP/FastMCP/FastMCPConfig.h
  24. 4
      Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/MCP/FastMCP/FastMCPManager.h
  25. 6
      Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/MCP/MCPBaseManager.h
  26. 7
      Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/MCP/MCPUnrealCommand.h
  27. 159
      Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/Tools/OpenAiModerator.h
  28. 6
      Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/Tools/OpenAiResponder.h
  29. 6
      Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/Tools/OpenRouterResponder.h
  30. 4
      Unreal/Plugins/AvatarCore_Manager/AvatarCore_Manager.uplugin
  31. BIN
      Unreal/Plugins/AvatarCore_Manager/Content/AvatarCoreManager.uasset
  32. BIN
      Unreal/Plugins/AvatarCore_Manager/Content/StateManagement/States/BP_BaseState.uasset
  33. BIN
      Unreal/Plugins/AvatarCore_Manager/Content/UnrealCommands/AICommand_CurrentLocation.uasset
  34. BIN
      Unreal/Plugins/AvatarCore_Manager/Content/UnrealCommands/DebutWait_Command.uasset
  35. BIN
      Unreal/Plugins/AvatarCore_Manager/Content/Widgets/Debug/Pages/ChildWidget/W_AutoTranslator.uasset
  36. BIN
      Unreal/Plugins/AvatarCore_Manager/Content/Widgets/Debug/Pages/W_DebugAvatarCoreSTT.uasset
  37. 1
      Unreal/Plugins/AvatarCore_Manager/Source/AvatarCore_Manager/AvatarCore_Manager.Build.cs
  38. 141
      Unreal/Plugins/AvatarCore_Manager/Source/AvatarCore_Manager/Private/FL_AvatarCoreManager.cpp
  39. 47
      Unreal/Plugins/AvatarCore_Manager/Source/AvatarCore_Manager/Public/AvatarCore_ManagerEnums.h
  40. 6
      Unreal/Plugins/AvatarCore_Manager/Source/AvatarCore_Manager/Public/FL_AvatarCoreManager.h
  41. 4
      Unreal/Plugins/AvatarCore_STT/AvatarCore_STT.uplugin
  42. 1
      Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/AvatarCore_STT.Build.cs
  43. 94
      Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Processor/Azure/STTProcessorAzure.cpp
  44. 20
      Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Processor/Parakeet/STTParakeetProcessorBase.cpp
  45. 16
      Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Processor/Whisper/STTProcessorWhisper.cpp
  46. 2
      Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/STTManagerBase.cpp
  47. 16
      Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Processor/Azure/STTAzureProcessorConfig.h
  48. 4
      Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Processor/Azure/STTProcessorAzure.h
  49. 30
      Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Processor/Parakeet/STTParakeetProcessorConfig.h
  50. 7
      Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Processor/STTBaseProcessorConfig.h
  51. 25
      Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Processor/Whisper/STTWhisperProcessorConfig.h
  52. 2
      Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/STTManagerBase.h
  53. 51
      Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/STTStructs.h
  54. 24
      Unreal/Plugins/AvatarCore_Shared/AvatarCore_Shared.uplugin
  55. BIN
      Unreal/Plugins/AvatarCore_Shared/Resources/Icon128.png
  56. 53
      Unreal/Plugins/AvatarCore_Shared/Source/AvatarCore_Shared/AvatarCore_Shared.Build.cs
  57. 20
      Unreal/Plugins/AvatarCore_Shared/Source/AvatarCore_Shared/Private/AvatarCore_Shared.cpp
  58. 27
      Unreal/Plugins/AvatarCore_Shared/Source/AvatarCore_Shared/Private/FL_AvatarCoreShared.cpp
  59. 62
      Unreal/Plugins/AvatarCore_Shared/Source/AvatarCore_Shared/Public/AvatarCoreSharedEnums.h
  60. 14
      Unreal/Plugins/AvatarCore_Shared/Source/AvatarCore_Shared/Public/AvatarCore_Shared.h
  61. 27
      Unreal/Plugins/AvatarCore_Shared/Source/AvatarCore_Shared/Public/FL_AvatarCoreShared.h
  62. 4
      Unreal/Plugins/AvatarCore_TTS/AvatarCore_TTS.uplugin
  63. 9
      Unreal/Plugins/AvatarCore_TTS/Source/.claude/settings.local.json
  64. 1
      Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/AvatarCore_TTS.Build.cs
  65. 62
      Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Private/Cartesia/CartesiaTTSManager.cpp
  66. 90
      Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Private/RealtimeAPI/RealtimeAPI_TTSManager.cpp
  67. 331
      Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Private/TTSManagerBase.cpp
  68. 35
      Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Private/TTSWebRTCChannel.cpp
  69. 9
      Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Public/RealtimeAPI/RealtimeAPI_TTSManager.h
  70. 73
      Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Public/TTSBaseConfig.h
  71. 36
      Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Public/TTSManagerBase.h
  72. 3
      Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Public/TTSWebRTCChannel.h
  73. 2
      Unreal/Plugins/BLogger/Source/BLogger/Public/SLogWidgetState.h
  74. BIN
      Unreal/Plugins/BTools/Content/Components/CheckProcessComponent.uasset
  75. 30
      Unreal/Plugins/BTools/Source/BTools/Private/BToolsBPLibrary.cpp
  76. 3
      Unreal/Plugins/BTools/Source/BTools/Public/BToolsBPLibrary.h
  77. 4
      Unreal/SPIE_Avatar.uproject
  78. 2
      Unreal/SyncAvatarCore.bat
  79. 109
      Unreal/SyncBackToAvatarCore.bat

1
Unreal/Config/DefaultInput.ini

@ -110,6 +110,7 @@ DoubleClickTime=0.200000
+ActionMappings=(ActionName="State7",bShift=False,bCtrl=False,bAlt=False,bCmd=False,Key=Seven)
+ActionMappings=(ActionName="State8",bShift=False,bCtrl=False,bAlt=False,bCmd=False,Key=Eight)
+ActionMappings=(ActionName="State9",bShift=False,bCtrl=False,bAlt=False,bCmd=False,Key=Nine)
+ActionMappings=(ActionName="Button",bShift=False,bCtrl=False,bAlt=False,bCmd=False,Key=GenericUSBController_Button12)
DefaultPlayerInputClass=/Script/EnhancedInput.EnhancedPlayerInput
DefaultInputComponentClass=/Script/EnhancedInput.EnhancedInputComponent
DefaultTouchInterface=/Engine/MobileResources/HUD/DefaultVirtualJoysticks.DefaultVirtualJoysticks

BIN
Unreal/Content/Project/BP/BP_Project_Manager.uasset (Stored with Git LFS)

Binary file not shown.

BIN
Unreal/Content/Project/BP/EnumsAndStructs/S_DEMO_Settings.uasset (Stored with Git LFS)

Binary file not shown.

BIN
Unreal/Content/Project/Widgets/ChildWidgets/W_ToolcallEntry.uasset (Stored with Git LFS)

Binary file not shown.

BIN
Unreal/Content/Project/Widgets/Debug/W_DebugStateWidget.uasset (Stored with Git LFS)

Binary file not shown.

BIN
Unreal/Content/Project/Widgets/W_DialogueBox.uasset (Stored with Git LFS)

Binary file not shown.

BIN
Unreal/Content/Project/Widgets/W_Toolcall.uasset (Stored with Git LFS)

Binary file not shown.

BIN
Unreal/Content/Project/Widgets/W_Toolcall_Simple.uasset (Stored with Git LFS)

Binary file not shown.

BIN
Unreal/Content/SPIE/BP/BP_SPIE_Manager_Child.uasset (Stored with Git LFS)

Binary file not shown.

BIN
Unreal/Content/SPIE/BP/S_SPIE_ConfigSettings.uasset (Stored with Git LFS)

Binary file not shown.

132
Unreal/Content/Schema/Spie_Config.schema.json

@ -291,21 +291,123 @@
}
},
{
"AzureSpeechService_API":
"STTAzureSettings":
{
"type": "string",
"tooltip": "Encrypted Azure API Key",
"default": "0Gc0wvOF2tTCr4APvZkDaieKCGeBR9c5EbC5utXgVWZUqu4IAEf6NQ721iHNu74SwfDoYSBGJHLmCbcXVDP+F4HKnwsfHr7WYi9Gv+CjJ7/UrOygkqlrP05hbHBrJiPLWDv4Gw==",
"category": "STT Settings"
"type": "struct",
"tooltip": "Settigns to configure the Mircrosoft Azure STT module",
"fields":
{
"AzureAPIKey":
{
"type": "string"
},
"AzureRegion":
{
"type": "string"
}
},
"default":
{
"AzureSpeechService_Region":
"AzureAPIKey": "0Gc0wvOF2tTCr4APvZkDaieKCGeBR9c5EbC5utXgVWZUqu4IAEf6NQ721iHNu74SwfDoYSBGJHLmCbcXVDP+F4HKnwsfHr7WYi9Gv+CjJ7/UrOygkqlrP05hbHBrJiPLWDv4Gw==",
"AzureRegion": "germanywestcentral"
},
"category": "STT"
}
},
{
"type": "string",
"tooltip": "Azure Region",
"default": "germanywestcentral",
"category": "STT Settings"
"STTWhisperSettings":
{
"type": "struct",
"tooltip": "Settigns to configure the OpenAI Whisper STT module",
"fields":
{
"OpenAI_API_Key":
{
"type": "string"
},
"WhisperURL":
{
"type": "string"
},
"Model":
{
"type": "enum",
"enum": [
"Whisper-1",
"4o Transcribe Mini",
"4o Transcribe"
],
"enumTypeName": "EOpenAITranscriptionModel"
},
"MinDuration":
{
"type": "float"
}
},
"default":
{
"OpenAI_API_Key": "UjzfgavJ45lCu+oB2vVAsKNbPT+k3XCv7t69Og6j0LmwxhD3OK5WDBxUvgKnuDrz3xuNHg==",
"WhisperURL": "api.openai.com/v1/audio/transcriptions",
"Model": "4o Transcribe",
"MinDuration": 0.75
},
"category": "STT"
}
},
{
"STTParakeetSettings":
{
"type": "struct",
"tooltip": "Settigns to configure the nVidia parakeet STT module",
"fields":
{
"PythonPath":
{
"type": "string"
},
"PretrainedModel":
{
"type": "string"
},
"Host":
{
"type": "string"
},
"Port":
{
"type": "integer"
},
"KeepAliveRule":
{
"type": "enum",
"enum": [
"Only keep Module open in Editor Mode",
"Never keep Module alive.",
"Keep Module alive in Editor and Shipping mode."
],
"enumTypeName": "EKeepAliveRule"
},
"Device":
{
"type": "string"
},
"UpdateIntervalSec":
{
"type": "float",
"tooltip": "How often (seconds) the Python server produces intermediate transcription updates"
}
},
"default":
{
"PythonPath": "python",
"PretrainedModel": "nvidia/parakeet-tdt-0.6b-v3",
"Host": "127.0.0.1",
"Port": 40200,
"KeepAliveRule": "Only keep Module open in Editor Mode",
"Device": "cuda:0",
"UpdateIntervalSec": 0.5
},
"category": "STT"
}
},
{
@ -562,7 +664,7 @@
"Marathi",
"Punjabi"
],
"itemsEnumTypeName": "ESTTLanguage"
"itemsEnumTypeName": "ELanguage"
},
"STTReplacements":
{
@ -785,11 +887,6 @@
"type": "struct",
"fields":
{
"bUseModeration":
{
"type": "boolean",
"tooltip": "Check user transcription for inappropriate behaviour first (adds a delay!)"
},
"bUseMCPServer":
{
"type": "boolean",
@ -826,7 +923,6 @@
},
"default":
{
"bUseModeration": false,
"bUseMCPServer": false,
"AIModelAudioOutput": true,
"MaxTokens": 1500,
@ -991,7 +1087,7 @@
"Marathi",
"Punjabi"
],
"enumTypeName": "ETTSLanguage"
"enumTypeName": "ELanguage"
}
},
"default":

6
Unreal/Plugins/AvatarCore_AI/AvatarCore_AI.uplugin

@ -5,7 +5,7 @@
"FriendlyName": "AvatarCore AI",
"Description": "A wrapper for OpenAI Assistents",
"Category": "Other",
"CreatedBy": "b.RexGmbh",
"CreatedBy": "b.ReX Gmbh",
"CreatedByURL": "",
"DocsURL": "",
"MarketplaceURL": "",
@ -29,6 +29,10 @@
{
"Name": "BTools",
"Enabled": true
},
{
"Name": "AvatarCore_Shared",
"Enabled": true
}
]
}

8
Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/AvatarCore_AI.Build.cs

@ -9,6 +9,11 @@ public class AvatarCore_AI : ModuleRules
{
PCHUsage = ModuleRules.PCHUsageMode.UseExplicitOrSharedPCHs;
if (Target.bBuildEditor)
{
PublicDependencyModuleNames.Add("UnrealEd");
}
PublicIncludePaths.AddRange(
new string[] {
// ... add public include paths required here ...
@ -51,7 +56,8 @@ public class AvatarCore_AI : ModuleRules
"SSL",
"Json",
"Projects", // Required for IPluginManager
"BTools"
"BTools",
"AvatarCore_Shared",
// ... add private dependencies that you statically link with here ...
}
);

90
Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/AIBaseManager.cpp

@ -1,6 +1,7 @@
// Fill out your copyright notice in the Description page of Project Settings.
#include "AIBaseManager.h"
#include "FL_AvatarCoreShared.h"
#include "Async/Async.h"
void UAIBaseManager::InitAIManager(UAIBaseConfig* AIConfig, bool DebugMode, AActor* InWorldReferenceActor)
@ -77,6 +78,11 @@ void UAIBaseManager::InitAIManagerChild(UAIBaseConfig* AIConfig, AActor* InWorld
SetNewState(EAvatarCoreAIState::Ready);
}
void UAIBaseManager::SetAILanguage(ELanguage NewLanguage)
{
ForcedLanguage = NewLanguage;
}
void UAIBaseManager::DeinitAIManager()
{
PreviousMessages.Empty();
@ -166,7 +172,13 @@ void UAIBaseManager::SendResponse(FAIMessage Message, bool NotifyDelay)
BroadcastAILog(FString::Printf(TEXT("AI Manager sent question/response: %s"), *Message.Message));
if (NotifyDelay)
UAIBaseManager::StartDelayedAnswerTimer();
//Add a fixed language tag. But only to the current message not to the stack. Do we really need this? Or does the system prompt is already enough?
FAIMessage currentMessage = Message;
if (ForcedLanguage != ELanguage::NONE && currentMessage.Role == EAvatarCoreAIPromptRole::User)
currentMessage.Message = "[Answer in " + UFL_AvatarCoreShared::LanguageToString(ForcedLanguage) + "]" + currentMessage.Message;
SendResponseChild(Message, NotifyDelay);
if (Message.Role == EAvatarCoreAIPromptRole::User)
AddMessageToArray(Message);
else if (Message.Role == EAvatarCoreAIPromptRole::Tool)
@ -186,10 +198,17 @@ void UAIBaseManager::RepeatText(FString TextToRepeat, bool DoRephrase)
AnswerCache.Empty();
ResponseID++;
FString Instruction;
if (DoRephrase)
if (DoRephrase) {
if(ForcedLanguage == ELanguage::NONE)
Instruction = "[REPHRASE] " + TextToRepeat;
else
Instruction = "[REPHRASE in " + UFL_AvatarCoreShared::LanguageToString(ForcedLanguage) + "] " + TextToRepeat;
}
else
if (ForcedLanguage == ELanguage::NONE)
Instruction = "[REPEAT] " + TextToRepeat;
else
Instruction = "[REPEAT in " + UFL_AvatarCoreShared::LanguageToString(ForcedLanguage) + "] " + TextToRepeat;
FAIMessage tmpPrompt;
tmpPrompt.Message = Instruction;
tmpPrompt.Role = EAvatarCoreAIPromptRole::System;
@ -317,7 +336,6 @@ void UAIBaseManager::RunMCPCommand(FString CommandName, FString Payload, FString
BroadcastAILog(FString::Printf(TEXT("Running Command '%s' with payload %s"), *CommandName, *Payload), true);
SetNewState(EAvatarCoreAIState::GettingInfo);
functionCallRunning = true;
if (FoundClass)
{
@ -329,7 +347,7 @@ void UAIBaseManager::RunMCPCommand(FString CommandName, FString Payload, FString
UMCPUnrealCommand* Cmd = NewObject<UMCPUnrealCommand>(this, FoundClass);
Cmd->Id = ToolCallId;
Cmd->SetWorldContext(WorldReferenceActor.Get());
ActiveCommands.Add(Cmd);
ActiveCommands.Add(ToolCallId, Cmd);
Cmd->OnCommandConfirmed.AddDynamic(this, &UAIBaseManager::CommandConfirmed);
Cmd->OnCommandDone.AddDynamic(this, &UAIBaseManager::CommandFinished);
@ -337,28 +355,38 @@ void UAIBaseManager::RunMCPCommand(FString CommandName, FString Payload, FString
Cmd->InitMCPCommand(World);
Cmd->Execute(World, Payload);
return;
}
if (MCPManager && MCPManager->HasCommand(CommandName))
{
if (!ToolCallId.IsEmpty())
MCPToolCallIds.Add(CommandName, ToolCallId);
MCPManager->ExecuteCommand(CommandName, Payload);
ActiveMCPCallIds.Add(ToolCallId);
MCPManager->ExecuteCommand(CommandName, Payload, ToolCallId);
}
OnAIToolcallStarted.Broadcast(CommandName, ToolCallId, Payload);
}
void UAIBaseManager::ClearMCPCommand()
{
for (UMCPUnrealCommand* Command : ActiveCommands)
for (auto& Pair : ActiveCommands)
{
if (Command)
if (Pair.Value)
{
Command->OnCommandDone.Clear();
Command->OnCommandFailed.Clear();
Pair.Value->OnCommandDone.Clear();
Pair.Value->OnCommandFailed.Clear();
}
}
ActiveCommands.Empty();
ActiveMCPCallIds.Empty();
}
bool UAIBaseManager::AnyToolcallRunning() const
{
return ActiveCommands.Num() > 0 || ActiveMCPCallIds.Num() > 0;
}
int UAIBaseManager::HowManyToolcallRunning() const
{
return ActiveCommands.Num() + ActiveMCPCallIds.Num();
}
FString UAIBaseManager::GetRoleAsString(EAvatarCoreAIPromptRole Role)
@ -384,13 +412,12 @@ void UAIBaseManager::CommandConfirmed(const FAIMessage& Message)
void UAIBaseManager::CommandFinished(const FAIMessage& Message)
{
ActiveCommands.RemoveAll([&Message](UMCPUnrealCommand* Cmd)
{
return Cmd && (Message.Id.IsEmpty() || Cmd->Id.Equals(Message.Id));
});
UMCPUnrealCommand* Cmd = ActiveCommands.FindRef(Message.Id);
FString ToolName = Cmd ? Cmd->GetCommandName() : TEXT("");
ActiveCommands.Remove(Message.Id);
OnAIToolcallFinished.Broadcast(ToolName, Message.Id, Message.Message);
SetNewState(EAvatarCoreAIState::Ready);
functionCallRunning = false;
if (bDebugMode)
BroadcastAILog(FString::Printf(TEXT("Command ran successfully. Answer: %s"), *Message.Message), true);
else
@ -412,29 +439,21 @@ void UAIBaseManager::CommandFinished(const FAIMessage& Message)
void UAIBaseManager::CommandFailed(const FAIMessage& Message)
{
ActiveCommands.RemoveAll([&Message](UMCPUnrealCommand* Cmd)
{
return Cmd && (Message.Id.IsEmpty() || Cmd->Id.Equals(Message.Id));
});
UMCPUnrealCommand* Cmd = ActiveCommands.FindRef(Message.Id);
FString ToolName = Cmd ? Cmd->GetCommandName() : TEXT("");
ActiveCommands.Remove(Message.Id);
functionCallRunning = false;
OnAIToolcallError.Broadcast(ToolName, Message.Id, Message.Message);
SetNewState(EAvatarCoreAIState::Ready);
BroadcastAILog(FString::Printf(TEXT("Command failed. Sending: %s"), *Message.Message), true);
SendResponse(Message, false);
}
void UAIBaseManager::MCPCommandFinished(const FString& Command, const FString& Payload)
{
FString FoundId;
FString* MCPId = MCPToolCallIds.Find(Command);
if (MCPId)
void UAIBaseManager::MCPCommandFinished(const FString& Command, const FString& Payload, const FString& ToolCallId)
{
FoundId = *MCPId;
MCPToolCallIds.Remove(Command);
}
ActiveMCPCallIds.Remove(ToolCallId);
OnAIToolcallFinished.Broadcast(Command, ToolCallId, Payload);
SetNewState(EAvatarCoreAIState::Ready);
functionCallRunning = false;
if (bDebugMode)
BroadcastAILog(FString::Printf(TEXT("MCP Command '%s' ran successfully. Answer: %s"), *Command, *Payload), true);
else
@ -442,10 +461,10 @@ void UAIBaseManager::MCPCommandFinished(const FString& Command, const FString& P
FAIMessage FinishedCommandMessage;
FinishedCommandMessage.Message = Payload;
if (!FoundId.IsEmpty())
if (!ToolCallId.IsEmpty())
{
FinishedCommandMessage.Role = EAvatarCoreAIPromptRole::Tool;
FinishedCommandMessage.Id = FoundId;
FinishedCommandMessage.Id = ToolCallId;
}
else
{
@ -454,9 +473,10 @@ void UAIBaseManager::MCPCommandFinished(const FString& Command, const FString& P
SendResponse(FinishedCommandMessage, false);
}
void UAIBaseManager::MCPCommandFailed(const FString& Command, const FString& Payload)
void UAIBaseManager::MCPCommandFailed(const FString& Command, const FString& Payload, const FString& ToolCallId)
{
functionCallRunning = false;
ActiveMCPCallIds.Remove(ToolCallId);
OnAIToolcallError.Broadcast(Command, ToolCallId, Payload);
SetNewState(EAvatarCoreAIState::Ready);
BroadcastAILog(FString::Printf(TEXT("MCP Command '%s' failed. Sending: %s"), *Command, *Payload), true);
FAIMessage FailedCommandMessage;

38
Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/MCP/FastMCP/FastMCPManager.cpp

@ -46,13 +46,30 @@ void UFastMCPManager::InitMCPManager(UMCPBaseConfig* InMCPConfig, bool DebugMode
return;
}
bool bKeepAlive = false;
#if WITH_EDITOR
if (FastMCPConfig->FastMCPSettings.KeepAliveRule == EKeepAliveRule::Never)
bKeepAlive = false;
else
bKeepAlive = true;
#else
if (FastMCPConfig->FastMCPSettings.KeepAliveRule == EKeepAliveRule::Always)
bKeepAlive = true;
else
bKeepAlive = false;
#endif
if(bKeepAlive)
{
bSpawnServerWhenNeeded = true;
CheckServerHealth();
#else
}
else
{
StopFastMCPServer();
StartFastMCPServer();
#endif
}
}
@ -83,7 +100,7 @@ void UFastMCPManager::StartFastMCPServer()
LogMessage(TEXT("Starting FastMCP Server..."));
// Build command line arguments
FString Arguments = FString::Printf(TEXT("\"%s\" \"%s\""), *FastMCPConfig->PythonPath, *FastMCPConfig->Arguments);
FString Arguments = FString::Printf(TEXT("\"%s\" \"%s\""), *FastMCPConfig->FastMCPSettings.PythonPath, *FastMCPConfig->FastMCPSettings.Arguments);
uint32 ProcId = 0;
@ -212,7 +229,7 @@ void UFastMCPManager::CheckServerHealth()
{
HealthCheckAttempts++;
if (HealthCheckAttempts > MaxHealthCheckAttempts)
if (HealthCheckAttempts > FastMCPConfig->FastMCPSettings.MaxHealthCheckTimeInSec)
{
LogError(TEXT("FastMCP Server failed to respond after maximum attempts"), EMCPManagerError::InitializationError);
if (UWorld* World = GEngine->GetWorldFromContextObject(this, EGetWorldErrorMode::LogAndReturnNull))
@ -409,7 +426,7 @@ bool UFastMCPManager::ParseToolsFromResponse(const FString& JsonResponse, TArray
return false;
}
void UFastMCPManager::ExecuteCommand(const FString& Command, const FString& Payload)
void UFastMCPManager::ExecuteCommand(const FString& Command, const FString& Payload, const FString& ToolCallId)
{
if (!bServerRunning)
{
@ -420,6 +437,7 @@ void UFastMCPManager::ExecuteCommand(const FString& Command, const FString& Payl
// Store current command for response handling
CurrentExecutingCommand = Command;
CurrentExecutingPayload = Payload;
CurrentToolCallId = ToolCallId;
SetState(EMCPManagerState::Busy);
LogMessage(FString::Printf(TEXT("Executing command: %s"), *Command));
@ -428,7 +446,7 @@ void UFastMCPManager::ExecuteCommand(const FString& Command, const FString& Payl
TSharedPtr<FJsonObject> CallMessage = MakeShareable(new FJsonObject);
CallMessage->SetStringField(TEXT("jsonrpc"), TEXT("2.0"));
CallMessage->SetStringField(TEXT("method"), TEXT("tools/call"));
CallMessage->SetNumberField(TEXT("id"), 3);
CallMessage->SetStringField(TEXT("id"), ToolCallId);
TSharedPtr<FJsonObject> Params = MakeShareable(new FJsonObject);
Params->SetStringField(TEXT("name"), Command);
@ -473,14 +491,14 @@ void UFastMCPManager::OnCommandExecuted(FHttpRequestPtr Request, FHttpResponsePt
if (!bWasSuccessful || !Response.IsValid())
{
LogError(FString::Printf(TEXT("Failed to execute command: %s"), *CurrentExecutingCommand), EMCPManagerError::ToolError);
OnMCPCommandFailed.Broadcast(CurrentExecutingCommand, CurrentExecutingPayload);
OnMCPCommandFailed.Broadcast(CurrentExecutingCommand, CurrentExecutingPayload, CurrentToolCallId);
return;
}
if (Response->GetResponseCode() != 200)
{
LogError(FString::Printf(TEXT("Command execution failed with code %d: %s"), Response->GetResponseCode(), *CurrentExecutingCommand), EMCPManagerError::ToolError);
OnMCPCommandFailed.Broadcast(CurrentExecutingCommand, CurrentExecutingPayload);
OnMCPCommandFailed.Broadcast(CurrentExecutingCommand, CurrentExecutingPayload, CurrentToolCallId);
return;
}
@ -499,7 +517,7 @@ void UFastMCPManager::OnCommandExecuted(FHttpRequestPtr Request, FHttpResponsePt
if ((*ResultObject)->HasField(TEXT("isError")) && (*ResultObject)->GetBoolField(TEXT("isError")))
{
LogError(FString::Printf(TEXT("Command execution failed: %s - %s"), *CurrentExecutingCommand, *ResponseContent), EMCPManagerError::ToolError);
OnMCPCommandFailed.Broadcast(CurrentExecutingCommand, ResponseContent);
OnMCPCommandFailed.Broadcast(CurrentExecutingCommand, ResponseContent, CurrentToolCallId);
return;
}
}
@ -508,7 +526,7 @@ void UFastMCPManager::OnCommandExecuted(FHttpRequestPtr Request, FHttpResponsePt
// Otherwise, broadcast success event with response
LogMessage(FString::Printf(TEXT("Command executed successfully: %s"), *CurrentExecutingCommand));
OnMCPCommandDone.Broadcast(CurrentExecutingCommand, ResponseContent);
OnMCPCommandDone.Broadcast(CurrentExecutingCommand, ResponseContent, CurrentToolCallId);
}
void UFastMCPManager::SendMCPMessage(const FString& JsonMessage, TFunction<void(FHttpRequestPtr, FHttpResponsePtr, bool)> OnComplete)

28
Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/MCP/MCPUnrealCommand.cpp

@ -5,6 +5,7 @@
void UMCPUnrealCommand::Execute_Implementation(UWorld* WorldReference, const FString& Payload)
{
SetWorldContext(WorldReference);
StartTimeout();
}
@ -114,9 +115,26 @@ UObject* UMCPUnrealCommand::GetWorldContextObject() const
UWorld* UMCPUnrealCommand::GetWorld() const
{
if (RequiredWorldContext)
return RequiredWorldContext->GetWorld();
if (const UObject* Outer = GetOuter())
return Outer->GetWorld();
return nullptr;
#if WITH_EDITOR
// Optional editor-only fallback before the game starts. This keeps the warning flood out of the way.
if (GEditor && HasAnyFlags(RF_ClassDefaultObject))
{
const FWorldContext& EditorWorldContext = GEditor->GetEditorWorldContext(false);
return EditorWorldContext.World();
}
// If it is not a ClassDefaultObject and still world is null, throw an error.
if (!RequiredWorldContext && !WarningMessageShown) {
FMessageLog("PIE").Error(FText::Format(
NSLOCTEXT("ObjectWithContext", "MissingWorldContext",
"WorldContextObject is not set on object '{0}' of class '{1}'."),
FText::FromString(GetName()),
FText::FromString(GetNameSafe(GetClass()))
));
FMessageLog("PIE").Notify();
WarningMessageShown = true;
}
#endif
return RequiredWorldContext ? RequiredWorldContext->GetWorld() : nullptr;
}

4
Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/RealtimeAPI/AvatarCoreAIRealtime.cpp

@ -545,12 +545,12 @@ void UAvatarCoreAIRealtime::OnWebSocketConnectionStringReceived(const FString& M
else if (RequestItem == EOpenAIRequestItem::audio_transcript || RequestItem == EOpenAIRequestItem::text)
ResponseTextDone = true;
if (functionCallRunning) {
if (AnyToolcallRunning()) {
return;
}
//float CurrentRequestDuration = (FDateTime::Now() - CurrentRequestStartTime).GetTotalSeconds();
if (CurrentRequestDuration < 0.25f && !functionCallRunning && !CurrentRequestID.IsEmpty())
if (CurrentRequestDuration < 0.25f && !AnyToolcallRunning() && !CurrentRequestID.IsEmpty())
{
CurrentRequestID.Empty();
if (CurrentRetries < MaxRetries)

178
Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/Tools/OpenAiModerator.cpp

@ -1,178 +0,0 @@
// Fill out your copyright notice in the Description page of Project Settings.
#include "Tools/OpenAiModerator.h"
#include "HttpModule.h"
#include "Interfaces/IHttpResponse.h"
#include "Dom/JsonObject.h"
#include "Serialization/JsonReader.h"
#include "Serialization/JsonSerializer.h"
FString UOpenAiModerator::StoredApiKey;
FString UOpenAiModerator::StoredModel = TEXT("omni-moderation-latest");
TArray<TWeakObjectPtr<UOpenAiModerator>> UOpenAiModerator::ActiveRequests;
void UOpenAiModerator::InitModeration(const FString& ApiKey, const FString& Model)
{
StoredApiKey = ApiKey;
StoredModel = Model;
}
UOpenAiModerator* UOpenAiModerator::UseModeration(UObject* WorldContextObject, const FString& Text)
{
UOpenAiModerator* Action = NewObject<UOpenAiModerator>();
Action->InputText = Text;
Action->RegisterWithGameInstance(WorldContextObject);
return Action;
}
void UOpenAiModerator::Activate()
{
if (StoredApiKey.IsEmpty())
{
UE_LOG(LogTemp, Error, TEXT("OpenAiModerator: API key not set. Call InitModeration first."));
FModerationResult EmptyResult;
Passed.Broadcast(EmptyResult);
return;
}
RequestStartTime = FPlatformTime::Seconds();
ActiveRequests.Add(this);
TSharedRef<IHttpRequest, ESPMode::ThreadSafe> HttpRequest = FHttpModule::Get().CreateRequest();
HttpRequest->SetURL(TEXT("https://api.openai.com/v1/moderations"));
HttpRequest->SetVerb(TEXT("POST"));
HttpRequest->SetHeader(TEXT("Content-Type"), TEXT("application/json"));
HttpRequest->SetHeader(TEXT("Authorization"), FString::Printf(TEXT("Bearer %s"), *StoredApiKey));
const FString Body = FString::Printf(TEXT("{\"model\":\"%s\",\"input\":\"%s\"}"),
*StoredModel,
*InputText.ReplaceCharWithEscapedChar());
HttpRequest->SetContentAsString(Body);
HttpRequest->OnProcessRequestComplete().BindUObject(this, &UOpenAiModerator::HandleHttpResponse);
if (!HttpRequest->ProcessRequest())
{
UE_LOG(LogTemp, Error, TEXT("OpenAiModerator: Failed to start HTTP request."));
FModerationResult EmptyResult;
Passed.Broadcast(EmptyResult);
}
}
void UOpenAiModerator::ClearModeration()
{
TArray<TWeakObjectPtr<UOpenAiModerator>> RequestsCopy = ActiveRequests;
ActiveRequests.Empty();
for (const TWeakObjectPtr<UOpenAiModerator>& Weak : RequestsCopy)
{
if (UOpenAiModerator* Action = Weak.Get())
{
Action->SetReadyToDestroy();
}
}
}
void UOpenAiModerator::HandleHttpResponse(FHttpRequestPtr Request, FHttpResponsePtr Response, bool bWasSuccessful)
{
ActiveRequests.RemoveAll([this](const TWeakObjectPtr<UOpenAiModerator>& Weak) { return !Weak.IsValid() || Weak.Get() == this; });
const float ElapsedTime = static_cast<float>(FPlatformTime::Seconds() - RequestStartTime);
FModerationResult Result;
Result.ExecutionTime = ElapsedTime;
if (!bWasSuccessful || !Response.IsValid())
{
UE_LOG(LogTemp, Error, TEXT("OpenAiModerator: HTTP request failed."));
Passed.Broadcast(Result);
return;
}
const int32 Code = Response->GetResponseCode();
const FString Body = Response->GetContentAsString();
if (Code < 200 || Code >= 300)
{
UE_LOG(LogTemp, Error, TEXT("OpenAiModerator: HTTP %d — %s"), Code, *Body);
Passed.Broadcast(Result);
return;
}
TSharedPtr<FJsonObject> RootObject;
TSharedRef<TJsonReader<>> Reader = TJsonReaderFactory<>::Create(Body);
if (!FJsonSerializer::Deserialize(Reader, RootObject) || !RootObject.IsValid())
{
UE_LOG(LogTemp, Error, TEXT("OpenAiModerator: Failed to parse response JSON."));
Passed.Broadcast(Result);
return;
}
const TArray<TSharedPtr<FJsonValue>>* ResultsArray;
if (!RootObject->TryGetArrayField(TEXT("results"), ResultsArray) || ResultsArray->Num() == 0)
{
UE_LOG(LogTemp, Error, TEXT("OpenAiModerator: No results in response."));
Passed.Broadcast(Result);
return;
}
const TSharedPtr<FJsonObject>& FirstResult = (*ResultsArray)[0]->AsObject();
if (!FirstResult.IsValid())
{
UE_LOG(LogTemp, Error, TEXT("OpenAiModerator: Invalid result object."));
Passed.Broadcast(Result);
return;
}
Result.bFlagged = FirstResult->GetBoolField(TEXT("flagged"));
const TSharedPtr<FJsonObject>* CategoriesObject;
if (FirstResult->TryGetObjectField(TEXT("categories"), CategoriesObject))
{
FModerationCategories& C = Result.Categories;
C.bSexual = (*CategoriesObject)->GetBoolField(TEXT("sexual"));
C.bSexualMinors = (*CategoriesObject)->GetBoolField(TEXT("sexual/minors"));
C.bHarassment = (*CategoriesObject)->GetBoolField(TEXT("harassment"));
C.bHarassmentThreatening = (*CategoriesObject)->GetBoolField(TEXT("harassment/threatening"));
C.bHate = (*CategoriesObject)->GetBoolField(TEXT("hate"));
C.bHateThreatening = (*CategoriesObject)->GetBoolField(TEXT("hate/threatening"));
C.bIllicit = (*CategoriesObject)->GetBoolField(TEXT("illicit"));
C.bIllicitViolent = (*CategoriesObject)->GetBoolField(TEXT("illicit/violent"));
C.bSelfHarm = (*CategoriesObject)->GetBoolField(TEXT("self-harm"));
C.bSelfHarmIntent = (*CategoriesObject)->GetBoolField(TEXT("self-harm/intent"));
C.bSelfHarmInstructions = (*CategoriesObject)->GetBoolField(TEXT("self-harm/instructions"));
C.bViolence = (*CategoriesObject)->GetBoolField(TEXT("violence"));
C.bViolenceGraphic = (*CategoriesObject)->GetBoolField(TEXT("violence/graphic"));
}
const TSharedPtr<FJsonObject>* ScoresObject;
if (FirstResult->TryGetObjectField(TEXT("category_scores"), ScoresObject))
{
FModerationCategoryScores& S = Result.CategoryScores;
S.Sexual = (*ScoresObject)->GetNumberField(TEXT("sexual"));
S.SexualMinors = (*ScoresObject)->GetNumberField(TEXT("sexual/minors"));
S.Harassment = (*ScoresObject)->GetNumberField(TEXT("harassment"));
S.HarassmentThreatening = (*ScoresObject)->GetNumberField(TEXT("harassment/threatening"));
S.Hate = (*ScoresObject)->GetNumberField(TEXT("hate"));
S.HateThreatening = (*ScoresObject)->GetNumberField(TEXT("hate/threatening"));
S.Illicit = (*ScoresObject)->GetNumberField(TEXT("illicit"));
S.IllicitViolent = (*ScoresObject)->GetNumberField(TEXT("illicit/violent"));
S.SelfHarm = (*ScoresObject)->GetNumberField(TEXT("self-harm"));
S.SelfHarmIntent = (*ScoresObject)->GetNumberField(TEXT("self-harm/intent"));
S.SelfHarmInstructions = (*ScoresObject)->GetNumberField(TEXT("self-harm/instructions"));
S.Violence = (*ScoresObject)->GetNumberField(TEXT("violence"));
S.ViolenceGraphic = (*ScoresObject)->GetNumberField(TEXT("violence/graphic"));
}
if (Result.bFlagged)
{
Flagged.Broadcast(Result);
}
else
{
Passed.Broadcast(Result);
}
}

7
Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/Tools/OpenAiResponder.cpp

@ -12,12 +12,14 @@
FString UOpenAiResponder::StoredApiKey;
FString UOpenAiResponder::StoredModel = TEXT("gpt-4.1");
bool UOpenAiResponder::IsResponderReady = false;
TArray<TWeakObjectPtr<UOpenAiResponder>> UOpenAiResponder::ActiveRequests;
void UOpenAiResponder::InitResponder(const FString& ApiKey, const FString& Model)
{
StoredApiKey = ApiKey;
StoredModel = Model;
IsResponderReady = true;
}
UOpenAiResponder* UOpenAiResponder::CallOpenAiResponse(
@ -44,6 +46,11 @@ UOpenAiResponder* UOpenAiResponder::CallOpenAiResponse(
return Action;
}
bool UOpenAiResponder::IsOpenAIResponderReady()
{
return IsResponderReady;
}
void UOpenAiResponder::Activate()
{
if (StoredApiKey.IsEmpty())

7
Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/Tools/OpenRouterResponder.cpp

@ -15,6 +15,7 @@ FString UOpenRouterResponder::StoredModel = TEXT("openai/gpt-4.1");
FString UOpenRouterResponder::StoredSiteUrl;
FString UOpenRouterResponder::StoredSiteName;
TArray<TWeakObjectPtr<UOpenRouterResponder>> UOpenRouterResponder::ActiveRequests;
bool UOpenRouterResponder::IsResponderReady = false;
void UOpenRouterResponder::InitResponder(const FString& ApiKey, const FString& Model, const FString& SiteUrl, const FString& SiteName)
{
@ -22,6 +23,7 @@ void UOpenRouterResponder::InitResponder(const FString& ApiKey, const FString& M
StoredModel = Model;
StoredSiteUrl = SiteUrl;
StoredSiteName = SiteName;
IsResponderReady = true;
}
UOpenRouterResponder* UOpenRouterResponder::CallOpenRouterResponse(
@ -46,6 +48,11 @@ UOpenRouterResponder* UOpenRouterResponder::CallOpenRouterResponse(
return Action;
}
bool UOpenRouterResponder::IsOpenRouterResponderReady()
{
return IsResponderReady;
}
void UOpenRouterResponder::Activate()
{
if (StoredApiKey.IsEmpty())

4
Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/AIBaseConfig.h

@ -29,10 +29,6 @@ struct FGlobalAISettings
{
GENERATED_BODY()
// Check user transcription for inappropriate behaviour first (adds a delay!)
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|MCP", meta = (ExposeOnSpawn = "true"))
bool bUseModeration = false;
// Boot up a FastMCP Server on Startup (On close keep open in Editor Mode, kill in shipping)
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|MCP", meta = (ExposeOnSpawn = "true"))
bool bUseMCPServer = false;

43
Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/AIBaseManager.h

@ -6,6 +6,7 @@
#include "UObject/Object.h"
#include "AIBaseConfig.h"
#include "AvatarCoreAIEnumsAndStructs.h"
#include "AvatarCoreSharedEnums.h"
#include "MCP/MCPBaseManager.h"
#include "AIBaseManager.generated.h"
@ -19,6 +20,9 @@ DECLARE_DYNAMIC_MULTICAST_DELEGATE_TwoParams(FOnAIError, FString, ErrorMessage,
DECLARE_DYNAMIC_MULTICAST_DELEGATE_TwoParams(FMulticastDelegateRealtimeAPIAudioChunk, const TArray<uint8>, PCMData, bool, IsFinal);
DECLARE_DYNAMIC_MULTICAST_DELEGATE(FOnAIDelayedAnswer);
DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FOnAIActivationStateChanged, bool, IsActive);
DECLARE_DYNAMIC_MULTICAST_DELEGATE_ThreeParams(FOnAIToolcallStarted, FString, ToolName, FString, ToolCallId, FString, Payload);
DECLARE_DYNAMIC_MULTICAST_DELEGATE_ThreeParams(FOnAIToolcallFinished, FString, ToolName, FString, ToolCallId, FString, Result);
DECLARE_DYNAMIC_MULTICAST_DELEGATE_ThreeParams(FOnAIToolcallError, FString, ToolName, FString, ToolCallId, FString, ErrorMessage);
class UMCPManager;
@ -57,6 +61,18 @@ public:
UPROPERTY(BlueprintAssignable, Category = "AvatarCoreAI|Events")
FOnAIActivationStateChanged OnAIActivationStateChanged;
// Fired when a tool/function call is dispatched (Payload is raw JSON input)
UPROPERTY(BlueprintAssignable, Category = "AvatarCoreAI|Events")
FOnAIToolcallStarted OnAIToolcallStarted;
// Fired when a tool/function call completes successfully (Result is raw JSON output)
UPROPERTY(BlueprintAssignable, Category = "AvatarCoreAI|Events")
FOnAIToolcallFinished OnAIToolcallFinished;
// Fired when a tool/function call fails
UPROPERTY(BlueprintAssignable, Category = "AvatarCoreAI|Events")
FOnAIToolcallError OnAIToolcallError;
/**
* Initializes the AI Manager with the given config and adds this UObject to the root set.
@ -67,6 +83,9 @@ public:
UFUNCTION(BlueprintCallable, Category = "AvatarCoreAI")
virtual void InitAIManagerChild(UAIBaseConfig* AIConfig, AActor* InWorldReferenceActor);
UFUNCTION(BlueprintCallable, Category = "AvatarCoreAI")
void SetAILanguage(ELanguage NewLanguage);
/** Returns the actor used as world context for commands */
UFUNCTION(BlueprintPure, Category = "AvatarCoreAI")
AActor* GetWorldReferenceActor() const { return WorldReferenceActor.Get(); }
@ -196,6 +215,14 @@ public:
UFUNCTION()
void OnRequestTimeout();
// Returns true if any Unreal command instance is active or a function call is in progress
UFUNCTION(BlueprintPure, Category = "AvatarCoreAI")
bool AnyToolcallRunning() const;
// Returns amount of toolcalls (Unreal Command and MCP) that are currenlty active
UFUNCTION(BlueprintPure, Category = "AvatarCoreAI")
int HowManyToolcallRunning() const;
// Add a command at runtime (handles AddToRoot)
UFUNCTION(BlueprintCallable, Category = "AvatarCoreAI|MCP")
UMCPBaseManager* GetMCPManager();
@ -242,11 +269,11 @@ protected:
/** Bound to MCPManager::OnMCPCommandDone — constructs FAIMessage from raw strings. */
UFUNCTION()
void MCPCommandFinished(const FString& Command, const FString& Payload);
void MCPCommandFinished(const FString& Command, const FString& Payload, const FString& ToolCallId);
/** Bound to MCPManager::OnMCPCommandFailed. */
UFUNCTION()
void MCPCommandFailed(const FString& Command, const FString& Payload);
void MCPCommandFailed(const FString& Command, const FString& Payload, const FString& ToolCallId);
//Add System/User/Assistant Message to memory archive
void AddMessageToArray(FAIMessage NewMessage);
@ -285,11 +312,10 @@ protected:
TArray<TSubclassOf<UMCPUnrealCommand>> UnrealCommandClasses;
TArray<FMCPToolInfo> UnrealCommandsToolInfos;
// Maps MCP server command name → tool_call_id for propagation through MCPCommandFinished
TMap<FString, FString> MCPToolCallIds;
UPROPERTY()
TArray<UMCPUnrealCommand*> ActiveCommands;
TMap<FString, UMCPUnrealCommand*> ActiveCommands; // ToolCallId → Unreal command
TSet<FString> ActiveMCPCallIds; // ToolCallIds of in-flight MCP server commands
/** MCP Manager for FastMCP server communication */
UPROPERTY()
@ -310,9 +336,6 @@ protected:
//Cached Answer
int ResponseID = 0;
//There is a function call in progress
bool functionCallRunning = false;
//Current State the AI Manager
EAvatarCoreAIState CurrentAIState = EAvatarCoreAIState::Disconnected;
@ -329,4 +352,6 @@ private:
TQueue<FAIMessage, EQueueMode::Spsc> ResponseQueue;
ELanguage ForcedLanguage = ELanguage::NONE;
};

30
Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/MCP/FastMCP/FastMCPConfig.h

@ -4,8 +4,31 @@
#include "CoreMinimal.h"
#include "MCP/MCPBaseConfig.h"
#include "AvatarCoreSharedEnums.h"
#include "FastMCPConfig.generated.h"
USTRUCT(BlueprintType)
struct FFastMCPSettings
{
GENERATED_BODY()
//Custom python environment - "python" will use the system default
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|MCP", meta = (ExposeOnSpawn = "true"))
FString PythonPath = "python";
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|MCP", meta = (ExposeOnSpawn = "true"))
EKeepAliveRule KeepAliveRule = EKeepAliveRule::EditorOnly;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|MCP", meta = (ExposeOnSpawn = "true"))
int MaxHealthCheckTimeInSec = 180;
//Additional arguments to pass
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|MCP", meta = (ExposeOnSpawn = "true"))
FString Arguments = "";
};
/**
*
*/
@ -20,12 +43,7 @@ public:
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|MCP", meta = (ExposeOnSpawn = "true"))
FString MCPExecutable = FPaths::ProjectContentDir() + "DB/FastMCP/FastMCPServer.bat";
//Custom python environment - "python" will use the system default
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreTTS|CoquiTTS", meta = (ExposeOnSpawn = "true"))
FString PythonPath = "python";
//Additional arguments to pass
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|MCP", meta = (ExposeOnSpawn = "true"))
FString Arguments = "";
FFastMCPSettings FastMCPSettings;
};

4
Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/MCP/FastMCP/FastMCPManager.h

@ -24,7 +24,7 @@ public:
virtual void InitMCPManager(UMCPBaseConfig* InMCPConfig, bool DebugMode) override;
virtual void DeinitMCPManager() override;
virtual void FetchAvailableTools() override;
virtual void ExecuteCommand(const FString& Command, const FString& Payload) override;
virtual void ExecuteCommand(const FString& Command, const FString& Payload, const FString& ToolCallId) override;
public:
// Helper to crop to first JSON object in a string (for SSE or noisy responses)
@ -67,9 +67,9 @@ private:
// Health check timer
FTimerHandle HealthCheckTimer;
int32 HealthCheckAttempts;
static constexpr int32 MaxHealthCheckAttempts = 90; // 90 seconds max wait
// Current command tracking
FString CurrentExecutingCommand;
FString CurrentExecutingPayload;
FString CurrentToolCallId;
};

6
Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/MCP/MCPBaseManager.h

@ -32,8 +32,8 @@ DECLARE_DYNAMIC_MULTICAST_DELEGATE_TwoParams(FOnMCPManagerError, const FString&,
DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FOnMCPManagerStateChanged, EMCPManagerState, NewMCPManagerState);
DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FOnMCPLog, const FString&, LogMessage);
DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FOnMCPToolsUpdated, const TArray<FMCPToolInfo>&, AvailableTools);
DECLARE_DYNAMIC_MULTICAST_DELEGATE_TwoParams(FOnMCPCommandDone, const FString&, Command, const FString&, Payload);
DECLARE_DYNAMIC_MULTICAST_DELEGATE_TwoParams(FOnMCPCommandFailed, const FString&, Command, const FString&, Payload);
DECLARE_DYNAMIC_MULTICAST_DELEGATE_ThreeParams(FOnMCPCommandDone, const FString&, Command, const FString&, Payload, const FString&, ToolCallId);
DECLARE_DYNAMIC_MULTICAST_DELEGATE_ThreeParams(FOnMCPCommandFailed, const FString&, Command, const FString&, Payload, const FString&, ToolCallId);
UCLASS(Abstract, Blueprintable, BlueprintType)
class AVATARCORE_AI_API UMCPBaseManager : public UObject
@ -81,7 +81,7 @@ public:
bool HasCommand(const FString& Command);
UFUNCTION(BlueprintCallable, Category = "AvatarCoreAI|MCP|Operations")
virtual void ExecuteCommand(const FString& Command, const FString& Payload) {};
virtual void ExecuteCommand(const FString& Command, const FString& Payload, const FString& ToolCallId) {};
// Blueprint Functions - State Management
UFUNCTION(BlueprintCallable, BlueprintPure, Category = "AvatarCoreAI|MCP|State")

7
Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/MCP/MCPUnrealCommand.h

@ -37,6 +37,8 @@ public:
UFUNCTION(BlueprintCallable, Category = "Context")
UObject* GetWorldContextObject() const;
virtual UWorld* GetWorld() const override;
// Result event (success)
UPROPERTY(BlueprintAssignable, Category = "Command")
FOnAICommandDone OnCommandDone;
@ -103,8 +105,9 @@ protected:
UFUNCTION(BlueprintCallable, Category = "Command")
AActor* GetActorOfClass(UWorld* World, TSubclassOf<AActor> ActorClass) const;
virtual UWorld* GetWorld() const override;
void StartTimeout();
void OnTimeout();
private:
mutable bool WarningMessageShown;
};

159
Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/Tools/OpenAiModerator.h

@ -1,159 +0,0 @@
// Fill out your copyright notice in the Description page of Project Settings.
#pragma once
#include "CoreMinimal.h"
#include "Kismet/BlueprintAsyncActionBase.h"
#include "Http.h"
#include "OpenAiModerator.generated.h"
USTRUCT(BlueprintType)
struct FModerationCategories
{
GENERATED_BODY()
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation")
bool bSexual = false;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation")
bool bSexualMinors = false;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation")
bool bHarassment = false;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation")
bool bHarassmentThreatening = false;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation")
bool bHate = false;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation")
bool bHateThreatening = false;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation")
bool bIllicit = false;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation")
bool bIllicitViolent = false;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation")
bool bSelfHarm = false;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation")
bool bSelfHarmIntent = false;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation")
bool bSelfHarmInstructions = false;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation")
bool bViolence = false;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation")
bool bViolenceGraphic = false;
};
USTRUCT(BlueprintType)
struct FModerationCategoryScores
{
GENERATED_BODY()
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation")
float Sexual = 0.f;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation")
float SexualMinors = 0.f;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation")
float Harassment = 0.f;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation")
float HarassmentThreatening = 0.f;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation")
float Hate = 0.f;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation")
float HateThreatening = 0.f;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation")
float Illicit = 0.f;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation")
float IllicitViolent = 0.f;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation")
float SelfHarm = 0.f;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation")
float SelfHarmIntent = 0.f;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation")
float SelfHarmInstructions = 0.f;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation")
float Violence = 0.f;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation")
float ViolenceGraphic = 0.f;
};
USTRUCT(BlueprintType)
struct FModerationResult
{
GENERATED_BODY()
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation")
bool bFlagged = false;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation")
FModerationCategories Categories;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation")
FModerationCategoryScores CategoryScores;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation")
float ExecutionTime = 0.f;
};
DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FOnModerationResult, const FModerationResult&, Result);
/**
* Async Blueprint node for OpenAI Moderation API.
* Call InitModeration once to set API key and model, then use UseModeration to classify text.
*/
UCLASS()
class AVATARCORE_AI_API UOpenAiModerator : public UBlueprintAsyncActionBase
{
GENERATED_BODY()
public:
/** Set API key and model. Call once before using UseModeration. */
UFUNCTION(BlueprintCallable, Category = "AvatarCoreAI|Moderation", meta = (DisplayName = "Init Moderation"))
static void InitModeration(const FString& ApiKey, const FString& Model = TEXT("omni-moderation-latest"));
/** Classify text using the OpenAI Moderation API. Returns via Flagged or Passed exec pins. */
UFUNCTION(BlueprintCallable, Category = "AvatarCoreAI|Moderation", meta = (BlueprintInternalUseOnly = "true", DisplayName = "Use Moderation", WorldContext = "WorldContextObject"))
static UOpenAiModerator* UseModeration(UObject* WorldContextObject, const FString& Text);
/** Cancel all active moderation requests. */
UFUNCTION(BlueprintCallable, Category = "AvatarCoreAI|Moderation", meta = (DisplayName = "Clear Moderation"))
static void ClearModeration();
UPROPERTY(BlueprintAssignable)
FOnModerationResult Flagged;
UPROPERTY(BlueprintAssignable)
FOnModerationResult Passed;
virtual void Activate() override;
private:
void HandleHttpResponse(FHttpRequestPtr Request, FHttpResponsePtr Response, bool bWasSuccessful);
FString InputText;
double RequestStartTime = 0.0;
static FString StoredApiKey;
static FString StoredModel;
static TArray<TWeakObjectPtr<UOpenAiModerator>> ActiveRequests;
};

6
Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/Tools/OpenAiResponder.h

@ -123,6 +123,10 @@ public:
bool bUseWebSearch = false,
EResponderToolChoice ToolChoice = EResponderToolChoice::None);
/** Is responder ready and api key set? */
UFUNCTION(BlueprintPure, Category = "AvatarCoreAI|Responder", meta = (DisplayName = "Is OpenAI Responder ready?"))
static bool IsOpenAIResponderReady();
/** Cancel all active response requests. */
UFUNCTION(BlueprintCallable, Category = "AvatarCoreAI|Responder", meta = (DisplayName = "Clear All Responses"))
static void ClearAllResponses();
@ -148,6 +152,8 @@ private:
EResponderToolChoice InputToolChoice = EResponderToolChoice::None;
double RequestStartTime = 0.0;
static bool IsResponderReady;
static FString StoredApiKey;
static FString StoredModel;
static TArray<TWeakObjectPtr<UOpenAiResponder>> ActiveRequests;

6
Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/Tools/OpenRouterResponder.h

@ -104,6 +104,10 @@ public:
ERouterReasoning Reasoning = ERouterReasoning::None,
ERouterToolChoice ToolChoice = ERouterToolChoice::Auto);
/** Is responder ready and api key set? */
UFUNCTION(BlueprintPure, Category = "AvatarCoreAI|Router", meta = (DisplayName = "Is OpenRouter Responder ready?"))
static bool IsOpenRouterResponderReady();
/** Cancel all active OpenRouter requests. */
UFUNCTION(BlueprintCallable, Category = "AvatarCoreAI|Router", meta = (DisplayName = "Clear All Router Responses"))
static void ClearAllResponses();
@ -128,6 +132,8 @@ private:
ERouterToolChoice InputToolChoice = ERouterToolChoice::Auto;
double RequestStartTime = 0.0;
static bool IsResponderReady;
static FString StoredApiKey;
static FString StoredModel;
static FString StoredSiteUrl;

4
Unreal/Plugins/AvatarCore_Manager/AvatarCore_Manager.uplugin

@ -37,6 +37,10 @@
{
"Name": "AvatarCore_TTS",
"Enabled": true
},
{
"Name": "AvatarCore_Shared",
"Enabled": true
}
]
}

BIN
Unreal/Plugins/AvatarCore_Manager/Content/AvatarCoreManager.uasset (Stored with Git LFS)

Binary file not shown.

BIN
Unreal/Plugins/AvatarCore_Manager/Content/StateManagement/States/BP_BaseState.uasset (Stored with Git LFS)

Binary file not shown.

BIN
Unreal/Plugins/AvatarCore_Manager/Content/UnrealCommands/AICommand_CurrentLocation.uasset (Stored with Git LFS)

Binary file not shown.

BIN
Unreal/Plugins/AvatarCore_Manager/Content/UnrealCommands/DebutWait_Command.uasset (Stored with Git LFS)

Binary file not shown.

BIN
Unreal/Plugins/AvatarCore_Manager/Content/Widgets/Debug/Pages/ChildWidget/W_AutoTranslator.uasset (Stored with Git LFS)

Binary file not shown.

BIN
Unreal/Plugins/AvatarCore_Manager/Content/Widgets/Debug/Pages/W_DebugAvatarCoreSTT.uasset (Stored with Git LFS)

Binary file not shown.

1
Unreal/Plugins/AvatarCore_Manager/Source/AvatarCore_Manager/AvatarCore_Manager.Build.cs

@ -42,6 +42,7 @@ public class AvatarCore_Manager : ModuleRules
"AvatarCore_TTS",
"AvatarCore_STT",
"AvatarCore_AI",
"AvatarCore_Shared",
"DeveloperSettings"
// ... add private dependencies that you statically link with here ...
}

141
Unreal/Plugins/AvatarCore_Manager/Source/AvatarCore_Manager/Private/FL_AvatarCoreManager.cpp

@ -17,144 +17,3 @@ void UFL_AvatarCoreManager::BindTTSToAIManager(UTTSManagerBase* TTSManager, UAIB
AIManager->OnAudioChunk.AddDynamic(TTSManager, &UTTSManagerBase::AddAudioChunk);
}
}
ETTSLanguage UFL_AvatarCoreManager::ConvertLanguageToTTS(ELanguage Language)
{
switch (Language)
{
case ELanguage::ar:
return ETTSLanguage::ar;
case ELanguage::de:
return ETTSLanguage::de;
case ELanguage::en:
return ETTSLanguage::en;
case ELanguage::fr:
return ETTSLanguage::fr;
case ELanguage::el:
return ETTSLanguage::el;
case ELanguage::hi:
return ETTSLanguage::hi;
case ELanguage::it:
return ETTSLanguage::it;
case ELanguage::ja:
return ETTSLanguage::ja;
case ELanguage::ko:
return ETTSLanguage::ko;
case ELanguage::zh:
return ETTSLanguage::zh;
case ELanguage::nl:
return ETTSLanguage::nl;
case ELanguage::pl:
return ETTSLanguage::pl;
case ELanguage::pt:
return ETTSLanguage::pt;
case ELanguage::ro:
return ETTSLanguage::ro;
case ELanguage::ru:
return ETTSLanguage::ru;
case ELanguage::es:
return ETTSLanguage::es;
case ELanguage::cs:
return ETTSLanguage::cs;
case ELanguage::tr:
return ETTSLanguage::tr;
case ELanguage::uk:
return ETTSLanguage::uk;
case ELanguage::hu:
return ETTSLanguage::hu;
case ELanguage::NONE:
default:
return ETTSLanguage::NONE;
}
}
ESTTLanguage UFL_AvatarCoreManager::ConvertLanguageToSTT(ELanguage Language)
{
switch (Language)
{
case ELanguage::ar:
return ESTTLanguage::ar;
case ELanguage::de:
return ESTTLanguage::de;
case ELanguage::en:
return ESTTLanguage::en;
case ELanguage::fr:
return ESTTLanguage::fr;
case ELanguage::el:
return ESTTLanguage::el;
case ELanguage::hi:
return ESTTLanguage::hi;
case ELanguage::it:
return ESTTLanguage::it;
case ELanguage::ja:
return ESTTLanguage::ja;
case ELanguage::ko:
return ESTTLanguage::ko;
case ELanguage::zh:
return ESTTLanguage::zh;
case ELanguage::nl:
return ESTTLanguage::nl;
case ELanguage::pl:
return ESTTLanguage::pl;
case ELanguage::pt:
return ESTTLanguage::pt;
case ELanguage::ro:
return ESTTLanguage::ro;
case ELanguage::ru:
return ESTTLanguage::ru;
case ELanguage::es:
return ESTTLanguage::es;
case ELanguage::cs:
return ESTTLanguage::cs;
case ELanguage::tr:
return ESTTLanguage::tr;
case ELanguage::uk:
return ESTTLanguage::uk;
case ELanguage::hu:
return ESTTLanguage::hu;
case ELanguage::NONE:
default:
return ESTTLanguage::NONE;
}
}

47
Unreal/Plugins/AvatarCore_Manager/Source/AvatarCore_Manager/Public/AvatarCore_ManagerEnums.h

@ -33,50 +33,3 @@ enum class EAvatarState : uint8 {
IdleActive UMETA(DisplayName = "Avatar idle active"),
IdlePassive UMETA(DisplayName = "Avatar idle passive")
};
UENUM(BlueprintType)
enum class ELanguage : uint8 {
NONE UMETA(DisplayName = "Unset"),
en UMETA(DisplayName = "English"),
fr UMETA(DisplayName = "French"),
de UMETA(DisplayName = "German"),
es UMETA(DisplayName = "Spanish"),
pt UMETA(DisplayName = "Portuguese"),
zh UMETA(DisplayName = "Chinese"),
ja UMETA(DisplayName = "Japanese"),
hi UMETA(DisplayName = "Hindi"),
it UMETA(DisplayName = "Italian"),
ko UMETA(DisplayName = "Korean"),
nl UMETA(DisplayName = "Dutch"),
pl UMETA(DisplayName = "Polish"),
ru UMETA(DisplayName = "Russian"),
sv UMETA(DisplayName = "Swedish"),
tr UMETA(DisplayName = "Turkish"),
tl UMETA(DisplayName = "Filipino"),
bg UMETA(DisplayName = "Bulgarian"),
ro UMETA(DisplayName = "Romanian"),
ar UMETA(DisplayName = "Arabic"),
cs UMETA(DisplayName = "Czech"),
el UMETA(DisplayName = "Greek"),
fi UMETA(DisplayName = "Finnish"),
hr UMETA(DisplayName = "Croatian"),
ms UMETA(DisplayName = "Malay"),
sk UMETA(DisplayName = "Slovak"),
da UMETA(DisplayName = "Danish"),
ta UMETA(DisplayName = "Tamil"),
uk UMETA(DisplayName = "Ukrainian"),
hu UMETA(DisplayName = "Hungarian"),
no UMETA(DisplayName = "Norwegian"),
vi UMETA(DisplayName = "Vietnamese"),
bn UMETA(DisplayName = "Bengali"),
th UMETA(DisplayName = "Thai"),
he UMETA(DisplayName = "Hebrew"),
ka UMETA(DisplayName = "Georgian"),
id UMETA(DisplayName = "Indonesian"),
te UMETA(DisplayName = "Telugu"),
gu UMETA(DisplayName = "Gujarati"),
kn UMETA(DisplayName = "Kannada"),
ml UMETA(DisplayName = "Malayalam"),
mr UMETA(DisplayName = "Marathi"),
pa UMETA(DisplayName = "Punjabi"),
};

6
Unreal/Plugins/AvatarCore_Manager/Source/AvatarCore_Manager/Public/FL_AvatarCoreManager.h

@ -48,10 +48,4 @@ class AVATARCORE_MANAGER_API UFL_AvatarCoreManager : public UBlueprintFunctionLi
UFUNCTION(BlueprintCallable, Category = "AvatarCoreManager")
static void BindTTSToAIManager(UTTSManagerBase* TTSManager, UAIBaseManager* AIManager);
UFUNCTION(BlueprintCallable, Category = "AvatarCoreManager")
static ETTSLanguage ConvertLanguageToTTS(ELanguage Language);
UFUNCTION(BlueprintCallable, Category = "AvatarCoreManager")
static ESTTLanguage ConvertLanguageToSTT(ELanguage Language);
};

4
Unreal/Plugins/AvatarCore_STT/AvatarCore_STT.uplugin

@ -29,6 +29,10 @@
{
"Name": "AudioCapture",
"Enabled": true
},
{
"Name": "AvatarCore_Shared",
"Enabled": true
}
]
}

1
Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/AvatarCore_STT.Build.cs

@ -101,6 +101,7 @@ public class AvatarCore_STT : ModuleRules
"Json",
"JsonUtilities",
"WebRTC",
"AvatarCore_Shared",
// ... add private dependencies that you statically link with here ...
}
);

94
Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Processor/Azure/STTProcessorAzure.cpp

@ -21,14 +21,14 @@ void USTTProcessorAzure::InitSTTProcessor(USTTManagerBase* BaseSTTManager, USTTB
return;
}
if (AzureProcessorConfig->AzureAPIKey.IsEmpty()) {
if (AzureProcessorConfig->AzureSettings.AzureAPIKey.IsEmpty()) {
STTManager->OnSTTError.Broadcast(TEXT("Azure API Key not set. Needs to be done before initializing modules."));
return;
}
// Convert FString to std::string
std::string SubscriptionKey = TCHAR_TO_UTF8(*AzureProcessorConfig->AzureAPIKey);
std::string Region = TCHAR_TO_UTF8(*AzureProcessorConfig->AzureRegion);
std::string SubscriptionKey = TCHAR_TO_UTF8(*AzureProcessorConfig->AzureSettings.AzureAPIKey);
std::string Region = TCHAR_TO_UTF8(*AzureProcessorConfig->AzureSettings.AzureRegion);
// Create the SpeechConfig object
config = SpeechSDK::SpeechConfig::FromSubscription(SubscriptionKey, Region);
@ -89,7 +89,7 @@ void USTTProcessorAzure::OnChunkReceived(TArray<int16> PCMData, FAudioInformatio
StopRecognition(false);
}
void USTTProcessorAzure::ChangeAzureLanguage(TArray<ESTTLanguage> InLanguages)
void USTTProcessorAzure::ChangeAzureLanguage(TArray<ELanguage> InLanguages)
{
AzureProcessorConfig->BaseSettings.STTLanguages = InLanguages;
if (bDebugMode && IsValid(STTManager))
@ -232,52 +232,52 @@ void USTTProcessorAzure::OnAzureError(FString Error)
}
}
FString USTTProcessorAzure::AzureEnumToString(ESTTLanguage Language)
FString USTTProcessorAzure::AzureEnumToString(ELanguage Language)
{
switch (Language)
{
case ESTTLanguage::en: return "en-US";
case ESTTLanguage::de: return "de-DE";
case ESTTLanguage::fr: return "fr-FR";
case ESTTLanguage::es: return "es-ES";
case ESTTLanguage::pt: return "pt-BR";
case ESTTLanguage::zh: return "zh-CN";
case ESTTLanguage::ja: return "ja-JP";
case ESTTLanguage::hi: return "hi-IN";
case ESTTLanguage::it: return "it-IT";
case ESTTLanguage::ko: return "ko-KR";
case ESTTLanguage::nl: return "nl-NL";
case ESTTLanguage::pl: return "pl-PL";
case ESTTLanguage::ru: return "ru-RU";
case ESTTLanguage::sv: return "sv-SE";
case ESTTLanguage::tr: return "tr-TR";
case ESTTLanguage::tl: return "fil-PH";
case ESTTLanguage::bg: return "bg-BG";
case ESTTLanguage::ro: return "ro-RO";
case ESTTLanguage::ar: return "ar-SA";
case ESTTLanguage::cs: return "cs-CZ";
case ESTTLanguage::el: return "el-GR";
case ESTTLanguage::fi: return "fi-FI";
case ESTTLanguage::hr: return "hr-HR";
case ESTTLanguage::ms: return "ms-MY";
case ESTTLanguage::sk: return "sk-SK";
case ESTTLanguage::da: return "da-DK";
case ESTTLanguage::ta: return "ta-IN";
case ESTTLanguage::uk: return "uk-UA";
case ESTTLanguage::hu: return "hu-HU";
case ESTTLanguage::no: return "nb-NO";
case ESTTLanguage::vi: return "vi-VN";
case ESTTLanguage::bn: return "bn-IN";
case ESTTLanguage::th: return "th-TH";
case ESTTLanguage::he: return "he-IL";
case ESTTLanguage::ka: return "ka-GE";
case ESTTLanguage::id: return "id-ID";
case ESTTLanguage::te: return "te-IN";
case ESTTLanguage::gu: return "gu-IN";
case ESTTLanguage::kn: return "kn-IN";
case ESTTLanguage::ml: return "ml-IN";
case ESTTLanguage::mr: return "mr-IN";
case ESTTLanguage::pa: return "pa-IN";
case ELanguage::en: return "en-US";
case ELanguage::de: return "de-DE";
case ELanguage::fr: return "fr-FR";
case ELanguage::es: return "es-ES";
case ELanguage::pt: return "pt-BR";
case ELanguage::zh: return "zh-CN";
case ELanguage::ja: return "ja-JP";
case ELanguage::hi: return "hi-IN";
case ELanguage::it: return "it-IT";
case ELanguage::ko: return "ko-KR";
case ELanguage::nl: return "nl-NL";
case ELanguage::pl: return "pl-PL";
case ELanguage::ru: return "ru-RU";
case ELanguage::sv: return "sv-SE";
case ELanguage::tr: return "tr-TR";
case ELanguage::tl: return "fil-PH";
case ELanguage::bg: return "bg-BG";
case ELanguage::ro: return "ro-RO";
case ELanguage::ar: return "ar-SA";
case ELanguage::cs: return "cs-CZ";
case ELanguage::el: return "el-GR";
case ELanguage::fi: return "fi-FI";
case ELanguage::hr: return "hr-HR";
case ELanguage::ms: return "ms-MY";
case ELanguage::sk: return "sk-SK";
case ELanguage::da: return "da-DK";
case ELanguage::ta: return "ta-IN";
case ELanguage::uk: return "uk-UA";
case ELanguage::hu: return "hu-HU";
case ELanguage::no: return "nb-NO";
case ELanguage::vi: return "vi-VN";
case ELanguage::bn: return "bn-IN";
case ELanguage::th: return "th-TH";
case ELanguage::he: return "he-IL";
case ELanguage::ka: return "ka-GE";
case ELanguage::id: return "id-ID";
case ELanguage::te: return "te-IN";
case ELanguage::gu: return "gu-IN";
case ELanguage::kn: return "kn-IN";
case ELanguage::ml: return "ml-IN";
case ELanguage::mr: return "mr-IN";
case ELanguage::pa: return "pa-IN";
default: return "UNDEFINED";
}
}

20
Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Processor/Parakeet/STTParakeetProcessorBase.cpp

@ -36,12 +36,12 @@ void USTTParakeetProcessorBase::InitSTTProcessor(USTTManagerBase* BaseSTTManager
}
#if WITH_EDITOR
if (ParakeetConfig->KeepAliveRule == ESTTKeepAliveRule::Never)
if (ParakeetConfig->ParakeetSettings.KeepAliveRule == EKeepAliveRule::Never)
bKeepAlive = false;
else
bKeepAlive = true;
#else
if (ParakeetConfig->KeepAliveRule == ESTTKeepAliveRule::Always)
if (ParakeetConfig->ParakeetSettings.KeepAliveRule == EKeepAliveRule::Always)
bKeepAlive = true;
else
bKeepAlive = false;
@ -75,8 +75,8 @@ void USTTParakeetProcessorBase::InitSTTProcessor(USTTManagerBase* BaseSTTManager
TSharedRef<FInternetAddr> Addr = Subsystem->CreateInternetAddr();
bool bIsValid = false;
Addr->SetIp(*ParakeetConfig->Host, bIsValid);
Addr->SetPort(ParakeetConfig->Port > 0 ? ParakeetConfig->Port : ParakeetDefaultPort);
Addr->SetIp(*ParakeetConfig->ParakeetSettings.Host, bIsValid);
Addr->SetPort(ParakeetConfig->ParakeetSettings.Port > 0 ? ParakeetConfig->ParakeetSettings.Port : ParakeetDefaultPort);
if (!bIsValid)
{
STTManager->OnSTTError.Broadcast(TEXT("Invalid address for Parakeet"));
@ -315,9 +315,9 @@ void USTTParakeetProcessorBase::SendConfiguration()
TSharedPtr<FJsonObject> ConfigObj = MakeShared<FJsonObject>();
TSharedPtr<FJsonObject> ParamsObj = MakeShared<FJsonObject>();
ParamsObj->SetStringField(TEXT("pretrained_model"), ParakeetConfig->PretrainedModel);
ParamsObj->SetStringField(TEXT("device"), ParakeetConfig->Device);
ParamsObj->SetNumberField(TEXT("update_interval"), ParakeetConfig->UpdateIntervalSec);
ParamsObj->SetStringField(TEXT("pretrained_model"), ParakeetConfig->ParakeetSettings.PretrainedModel);
ParamsObj->SetStringField(TEXT("device"), ParakeetConfig->ParakeetSettings.Device);
ParamsObj->SetNumberField(TEXT("update_interval"), ParakeetConfig->ParakeetSettings.UpdateIntervalSec);
ConfigObj->SetStringField(TEXT("type"), TEXT("config"));
ConfigObj->SetObjectField(TEXT("params"), ParamsObj);
@ -429,9 +429,9 @@ bool USTTParakeetProcessorBase::StartParakeetProcess()
}
const FString Args = FString::Printf(TEXT("\"%s\" serve-config --host %s --port %d"),
*ParakeetConfig->PythonPath,
*ParakeetConfig->Host,
ParakeetConfig->Port > 0 ? ParakeetConfig->Port : ParakeetDefaultPort);
*ParakeetConfig->ParakeetSettings.PythonPath,
*ParakeetConfig->ParakeetSettings.Host,
ParakeetConfig->ParakeetSettings.Port > 0 ? ParakeetConfig->ParakeetSettings.Port : ParakeetDefaultPort);
uint32 ProcId = 0;

16
Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Processor/Whisper/STTProcessorWhisper.cpp

@ -51,7 +51,7 @@ void USTTProcessorWhisper::InitSTTProcessor(USTTManagerBase* BaseSTTManager, UST
return;
}
if (WhisperProcessorConfig->OpenAI_API_Key.IsEmpty()) {
if (WhisperProcessorConfig->WhisperSettings.OpenAI_API_Key.IsEmpty()) {
if (IsValid(STTManager))
STTManager->OnSTTError.Broadcast(TEXT("OpenAI API Key not set. Needs to be done before initializing modules."));
return;
@ -61,7 +61,7 @@ void USTTProcessorWhisper::InitSTTProcessor(USTTManagerBase* BaseSTTManager, UST
PerformHealthCheck();
STTManager->OnSTTLog.Broadcast(FString::Printf(TEXT("STTProcessor OpenAI %s initialized successfully."), *TranscribeModelEnumToString(WhisperProcessorConfig->Model)));
STTManager->OnSTTLog.Broadcast(FString::Printf(TEXT("STTProcessor OpenAI %s initialized successfully."), *TranscribeModelEnumToString(WhisperProcessorConfig->WhisperSettings.Model)));
}
void USTTProcessorWhisper::ClearSTTProcessor()
@ -114,7 +114,7 @@ void USTTProcessorWhisper::StartTranscriptionFromBuffer()
{
const float Frames = static_cast<float>(PCMDataCopy.Num()) / static_cast<float>(AudioInfoCopy.NumChannels);
const float DurationSeconds = Frames / static_cast<float>(AudioInfoCopy.SampleRate);
if (DurationSeconds < WhisperProcessorConfig->MinDuration)
if (DurationSeconds < WhisperProcessorConfig->WhisperSettings.MinDuration)
{
return;
}
@ -198,9 +198,9 @@ void USTTProcessorWhisper::BuildMultipartBody(const TArray<uint8>& WavData, cons
FString BoundaryLine = FString::Printf(TEXT("--%s\r\n"), *Boundary);
AppendStringToBody(OutBody, BoundaryLine);
AppendStringToBody(OutBody, TEXT("Content-Disposition: form-data; name=\"model\"\r\n\r\n"));
AppendStringToBody(OutBody, TranscribeModelEnumToString(WhisperProcessorConfig->Model) + TEXT("\r\n"));
AppendStringToBody(OutBody, TranscribeModelEnumToString(WhisperProcessorConfig->WhisperSettings.Model) + TEXT("\r\n"));
if (WhisperProcessorConfig->Model == EOpenAITranscriptionModel::Whisper1)
if (WhisperProcessorConfig->WhisperSettings.Model == EOpenAITranscriptionModel::Whisper1)
{
BoundaryLine = FString::Printf(TEXT("--%s\r\n"), *Boundary);
AppendStringToBody(OutBody, BoundaryLine);
@ -233,7 +233,7 @@ void USTTProcessorWhisper::NormalizeWhisperURL()
if (!WhisperProcessorConfig)
return;
NormalizedWhisperURL = WhisperProcessorConfig->WhisperURL;
NormalizedWhisperURL = WhisperProcessorConfig->WhisperSettings.WhisperURL;
if (!NormalizedWhisperURL.StartsWith(TEXT("http://")) && !NormalizedWhisperURL.StartsWith(TEXT("https://")))
{
NormalizedWhisperURL = FString::Printf(TEXT("https://%s"), *NormalizedWhisperURL);
@ -252,7 +252,7 @@ void USTTProcessorWhisper::PerformHealthCheck()
TSharedRef<IHttpRequest, ESPMode::ThreadSafe> Request = HttpModule.CreateRequest();
Request->SetURL(NormalizedWhisperURL);
Request->SetVerb(TEXT("GET"));
FString AuthHeader = FString::Printf(TEXT("Bearer %s"), *WhisperProcessorConfig->OpenAI_API_Key);
FString AuthHeader = FString::Printf(TEXT("Bearer %s"), *WhisperProcessorConfig->WhisperSettings.OpenAI_API_Key);
Request->SetHeader(TEXT("Authorization"), AuthHeader);
Request->OnProcessRequestComplete().BindLambda([
WeakManager = TWeakObjectPtr<USTTManagerBase>(STTManager)](FHttpRequestPtr Req, FHttpResponsePtr Resp, bool bWasSuccessful)
@ -292,7 +292,7 @@ void USTTProcessorWhisper::SendWhisperRequest(TArray<uint8>&& WavData)
Request->SetURL(NormalizedWhisperURL);
Request->SetVerb(TEXT("POST"));
FString AuthHeader = FString::Printf(TEXT("Bearer %s"), *WhisperProcessorConfig->OpenAI_API_Key);
FString AuthHeader = FString::Printf(TEXT("Bearer %s"), *WhisperProcessorConfig->WhisperSettings.OpenAI_API_Key);
Request->SetHeader(TEXT("Authorization"), AuthHeader);
Request->SetHeader(TEXT("Accept"), TEXT("application/json"));

2
Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/STTManagerBase.cpp

@ -175,7 +175,7 @@ bool USTTManagerBase::IsBlocked()
return (CurrentSpeechState==ESTTTalkingState::BLOCKED);
}
void USTTManagerBase::SetLanguage(TArray<ESTTLanguage> NewLanguages)
void USTTManagerBase::SetSTTLanguage(TArray<ELanguage> NewLanguages)
{
if (!IsValid(ProcessorConfig))
return;

16
Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Processor/Azure/STTAzureProcessorConfig.h

@ -6,6 +6,18 @@
#include "Processor/STTBaseProcessorConfig.h"
#include "STTAzureProcessorConfig.generated.h"
USTRUCT(BlueprintType)
struct FSTTAzureSettings
{
GENERATED_BODY()
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreSTT|Azure", meta = (ExposeOnSpawn = "true"))
FString AzureAPIKey = "";
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreSTT|Azure", meta = (ExposeOnSpawn = "true"))
FString AzureRegion = "germanywestcentral";
};
/**
*
*/
@ -19,7 +31,5 @@ public:
USTTAzureProcessorConfig(const FObjectInitializer& ObjectInitializer);
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreSTT|Azure", meta = (ExposeOnSpawn = "true"))
FString AzureAPIKey = "";
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreSTT|Azure", meta = (ExposeOnSpawn = "true"))
FString AzureRegion = "germanywestcentral";
FSTTAzureSettings AzureSettings;
};

4
Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Processor/Azure/STTProcessorAzure.h

@ -32,7 +32,7 @@ class AVATARCORE_STT_API USTTProcessorAzure : public USTTProcessorBase
public:
UFUNCTION(BlueprintCallable, Category = STTManager)
virtual void ChangeAzureLanguage(TArray<ESTTLanguage> InLanguages);
virtual void ChangeAzureLanguage(TArray<ELanguage> InLanguages);
private:
@ -62,7 +62,7 @@ public:
void OnAzureError(FString Error);
UFUNCTION(BlueprintPure, Category = STTManager)
FString AzureEnumToString(ESTTLanguage Language);
FString AzureEnumToString(ELanguage Language);
USTTAzureProcessorConfig* AzureProcessorConfig;
};

30
Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Processor/Parakeet/STTParakeetProcessorConfig.h

@ -6,18 +6,11 @@
#include "Processor/STTBaseProcessorConfig.h"
#include "STTParakeetProcessorConfig.generated.h"
/**
* Configuration for the Parakeet STT Processor (NVIDIA NeMo ASR).
*/
UCLASS(Blueprintable, BlueprintType)
class AVATARCORE_STT_API USTTParakeetProcessorConfig : public USTTBaseProcessorConfig
USTRUCT(BlueprintType)
struct FSTTParakeetSettings
{
GENERATED_BODY()
public:
USTTParakeetProcessorConfig(const FObjectInitializer& ObjectInitializer);
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreSTT|Parakeet", meta = (ExposeOnSpawn = "true"))
FString PythonPath = "python";
@ -31,7 +24,7 @@ public:
int32 Port = 40200;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreSTT|Parakeet", meta = (ExposeOnSpawn = "true"))
ESTTKeepAliveRule KeepAliveRule = ESTTKeepAliveRule::EditorOnly;
EKeepAliveRule KeepAliveRule = EKeepAliveRule::EditorOnly;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreSTT|Parakeet", meta = (ExposeOnSpawn = "true"))
FString Device = "cuda:0";
@ -39,4 +32,21 @@ public:
// How often (seconds) the Python server produces intermediate transcription updates
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreSTT|Parakeet", meta = (ExposeOnSpawn = "true", ClampMin = "0.1", ClampMax = "5.0"))
float UpdateIntervalSec = 0.5f;
};
/**
* Configuration for the Parakeet STT Processor (NVIDIA NeMo ASR).
*/
UCLASS(Blueprintable, BlueprintType)
class AVATARCORE_STT_API USTTParakeetProcessorConfig : public USTTBaseProcessorConfig
{
GENERATED_BODY()
public:
USTTParakeetProcessorConfig(const FObjectInitializer& ObjectInitializer);
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreSTT|Parakeet", meta = (ExposeOnSpawn = "true", ClampMin = "0.1", ClampMax = "5.0"))
FSTTParakeetSettings ParakeetSettings;
};

7
Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Processor/STTBaseProcessorConfig.h

@ -7,13 +7,6 @@
#include "STTStructs.h"
#include "STTBaseProcessorConfig.generated.h"
UENUM(BlueprintType)
enum class ESTTKeepAliveRule : uint8 {
EditorOnly UMETA(DisplayName = "Only keep STT Module open in Editor Mode"),
Never UMETA(DisplayName = "Never keep STT Module alive."),
Always UMETA(DisplayName = "Keep STT Module alive in Editor and Shipping mode.")
};
class USTTProcessorBase;
/**

25
Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Processor/Whisper/STTWhisperProcessorConfig.h

@ -14,6 +14,23 @@ enum class EOpenAITranscriptionModel : uint8
Transcribe4o UMETA(DisplayName = "4o Transcribe")
};
USTRUCT(BlueprintType)
struct FSTTWhisperSettings
{
GENERATED_BODY()
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreSTT|Whisper", meta = (ExposeOnSpawn = "true"))
FString OpenAI_API_Key = "";
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreSTT|Whisper", meta = (ExposeOnSpawn = "true"))
FString WhisperURL = "api.openai.com/v1/audio/transcriptions";
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreSTT|Whisper", meta = (ExposeOnSpawn = "true"))
EOpenAITranscriptionModel Model = EOpenAITranscriptionModel::Transcribe4o;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreSTT|Whisper", meta = (ExposeOnSpawn = "true"))
float MinDuration = 0.75f;
};
/**
*
*/
@ -27,12 +44,6 @@ public:
USTTWhisperProcessorConfig(const FObjectInitializer& ObjectInitializer);
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreSTT|Whisper", meta = (ExposeOnSpawn = "true"))
FString OpenAI_API_Key = "";
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreSTT|Whisper", meta = (ExposeOnSpawn = "true"))
FString WhisperURL = "api.openai.com/v1/audio/transcriptions";
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreSTT|Whisper", meta = (ExposeOnSpawn = "true"))
EOpenAITranscriptionModel Model = EOpenAITranscriptionModel::Transcribe4o;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreSTT|Whisper", meta = (ExposeOnSpawn = "true"))
float MinDuration = 0.75f;
FSTTWhisperSettings WhisperSettings;
};

2
Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/STTManagerBase.h

@ -99,7 +99,7 @@ public:
bool IsBlocked();
UFUNCTION(BlueprintCallable, Category = "AvatarCoreSTT")
void SetLanguage(TArray<ESTTLanguage> NewLanguages);
void SetSTTLanguage(TArray<ELanguage> NewLanguages);
UFUNCTION(BlueprintCallable, Category = "AvatarCoreSTT")
void AddSpecialWord(FString NewWord);

51
Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/STTStructs.h

@ -1,6 +1,7 @@
#pragma once
#include "CoreMinimal.h"
#include "AvatarCoreSharedEnums.h"
#include "STTStructs.generated.h"
USTRUCT(BlueprintType)
@ -70,54 +71,6 @@ enum class ESpeexDSPState : uint8
SPEEXPREPROCESS_SET_AGC_TARGET = 46 UMETA(DisplayName = "preprocessor Automatic Gain Control level (int32)")
};
UENUM(BlueprintType)
enum class ESTTLanguage : uint8
{
NONE UMETA(DisplayName = "Unset"),
en UMETA(DisplayName = "English"),
fr UMETA(DisplayName = "French"),
de UMETA(DisplayName = "German"),
es UMETA(DisplayName = "Spanish"),
pt UMETA(DisplayName = "Portuguese"),
zh UMETA(DisplayName = "Chinese"),
ja UMETA(DisplayName = "Japanese"),
hi UMETA(DisplayName = "Hindi"),
it UMETA(DisplayName = "Italian"),
ko UMETA(DisplayName = "Korean"),
nl UMETA(DisplayName = "Dutch"),
pl UMETA(DisplayName = "Polish"),
ru UMETA(DisplayName = "Russian"),
sv UMETA(DisplayName = "Swedish"),
tr UMETA(DisplayName = "Turkish"),
tl UMETA(DisplayName = "Filipino"),
bg UMETA(DisplayName = "Bulgarian"),
ro UMETA(DisplayName = "Romanian"),
ar UMETA(DisplayName = "Arabic"),
cs UMETA(DisplayName = "Czech"),
el UMETA(DisplayName = "Greek"),
fi UMETA(DisplayName = "Finnish"),
hr UMETA(DisplayName = "Croatian"),
ms UMETA(DisplayName = "Malay"),
sk UMETA(DisplayName = "Slovak"),
da UMETA(DisplayName = "Danish"),
ta UMETA(DisplayName = "Tamil"),
uk UMETA(DisplayName = "Ukrainian"),
hu UMETA(DisplayName = "Hungarian"),
no UMETA(DisplayName = "Norwegian"),
vi UMETA(DisplayName = "Vietnamese"),
bn UMETA(DisplayName = "Bengali"),
th UMETA(DisplayName = "Thai"),
he UMETA(DisplayName = "Hebrew"),
ka UMETA(DisplayName = "Georgian"),
id UMETA(DisplayName = "Indonesian"),
te UMETA(DisplayName = "Telugu"),
gu UMETA(DisplayName = "Gujarati"),
kn UMETA(DisplayName = "Kannada"),
ml UMETA(DisplayName = "Malayalam"),
mr UMETA(DisplayName = "Marathi"),
pa UMETA(DisplayName = "Punjabi"),
};
USTRUCT(BlueprintType)
struct FWebRTCSettings
{
@ -262,7 +215,7 @@ struct FSTTBaseSettings
UPROPERTY(EditAnywhere, BlueprintReadWrite, meta = (ToolTip = "Settings of the SpeexDSP Module", Category = "STT|Base"))
FSpeexDSPSettings SpeexDSPSettings;
UPROPERTY(EditAnywhere, BlueprintReadWrite, meta = (ToolTip = "All languages the STT module should understand simultaneously.", Category = "STT|Base"))
TArray<ESTTLanguage> STTLanguages = { ESTTLanguage::de, ESTTLanguage::en };
TArray<ELanguage> STTLanguages = { ELanguage::de, ELanguage::en };
UPROPERTY(EditAnywhere, BlueprintReadWrite, meta = (ToolTip = "Transcriptions to always change to another word.", Category = "STT|Base"))
TArray<FSTTWordReplacement> STTReplacements;
UPROPERTY(EditAnywhere, BlueprintReadWrite, meta = (ToolTip = "Special words that the transcription service needs to know (e.g. b.ReX or Bruce-B).", Category = "STT|Base"))

24
Unreal/Plugins/AvatarCore_Shared/AvatarCore_Shared.uplugin

@ -0,0 +1,24 @@
{
"FileVersion": 3,
"Version": 1,
"VersionName": "1.0",
"FriendlyName": "AvatarCore_Shared",
"Description": "Shared assets, structs and enums. In theory we want to be AI, TTS and STT to be separate.",
"Category": "Other",
"CreatedBy": "b.ReX GmbH",
"CreatedByURL": "",
"DocsURL": "",
"MarketplaceURL": "",
"SupportURL": "",
"CanContainContent": true,
"IsBetaVersion": false,
"IsExperimentalVersion": false,
"Installed": false,
"Modules": [
{
"Name": "AvatarCore_Shared",
"Type": "Runtime",
"LoadingPhase": "Default"
}
]
}

BIN
Unreal/Plugins/AvatarCore_Shared/Resources/Icon128.png (Stored with Git LFS)

Binary file not shown.

53
Unreal/Plugins/AvatarCore_Shared/Source/AvatarCore_Shared/AvatarCore_Shared.Build.cs

@ -0,0 +1,53 @@
// Copyright Epic Games, Inc. All Rights Reserved.
using UnrealBuildTool;
public class AvatarCore_Shared : ModuleRules
{
public AvatarCore_Shared(ReadOnlyTargetRules Target) : base(Target)
{
PCHUsage = ModuleRules.PCHUsageMode.UseExplicitOrSharedPCHs;
PublicIncludePaths.AddRange(
new string[] {
// ... add public include paths required here ...
}
);
PrivateIncludePaths.AddRange(
new string[] {
// ... add other private include paths required here ...
}
);
PublicDependencyModuleNames.AddRange(
new string[]
{
"Core",
// ... add other public dependencies that you statically link with here ...
}
);
PrivateDependencyModuleNames.AddRange(
new string[]
{
"CoreUObject",
"Engine",
"Slate",
"SlateCore",
// ... add private dependencies that you statically link with here ...
}
);
DynamicallyLoadedModuleNames.AddRange(
new string[]
{
// ... add any modules that your module loads dynamically here ...
}
);
}
}

20
Unreal/Plugins/AvatarCore_Shared/Source/AvatarCore_Shared/Private/AvatarCore_Shared.cpp

@ -0,0 +1,20 @@
// Copyright Epic Games, Inc. All Rights Reserved.
#include "AvatarCore_Shared.h"
#define LOCTEXT_NAMESPACE "FAvatarCore_SharedModule"
void FAvatarCore_SharedModule::StartupModule()
{
// This code will execute after your module is loaded into memory; the exact timing is specified in the .uplugin file per-module
}
void FAvatarCore_SharedModule::ShutdownModule()
{
// This function may be called during shutdown to clean up your module. For modules that support dynamic reloading,
// we call this function before unloading the module.
}
#undef LOCTEXT_NAMESPACE
IMPLEMENT_MODULE(FAvatarCore_SharedModule, AvatarCore_Shared)

27
Unreal/Plugins/AvatarCore_Shared/Source/AvatarCore_Shared/Private/FL_AvatarCoreShared.cpp

@ -0,0 +1,27 @@
// Fill out your copyright notice in the Description page of Project Settings.
#include "FL_AvatarCoreShared.h"
ELanguage UFL_AvatarCoreShared::StringToLanguage(const FString& InString)
{
const UEnum* EnumPtr = StaticEnum<ELanguage>();
if (!EnumPtr)
{
return ELanguage::NONE;
}
FString Normalized = InString.ToLower();
int64 Value = EnumPtr->GetValueByNameString(Normalized);
return Value != INDEX_NONE
? static_cast<ELanguage>(Value)
: ELanguage::NONE;
}
FString UFL_AvatarCoreShared::LanguageToString(ELanguage Language)
{
return StaticEnum<ELanguage>()->GetNameStringByValue((int64)Language);
}

62
Unreal/Plugins/AvatarCore_Shared/Source/AvatarCore_Shared/Public/AvatarCoreSharedEnums.h

@ -0,0 +1,62 @@
// Fill out your copyright notice in the Description page of Project Settings.
#pragma once
#include "CoreMinimal.h"
#include "UObject/NoExportTypes.h"
#include "AvatarCoreSharedEnums.generated.h"
UENUM(BlueprintType)
enum class ELanguage : uint8 {
NONE UMETA(DisplayName = "Unset"),
en UMETA(DisplayName = "English"),
fr UMETA(DisplayName = "French"),
de UMETA(DisplayName = "German"),
es UMETA(DisplayName = "Spanish"),
pt UMETA(DisplayName = "Portuguese"),
zh UMETA(DisplayName = "Chinese"),
ja UMETA(DisplayName = "Japanese"),
hi UMETA(DisplayName = "Hindi"),
it UMETA(DisplayName = "Italian"),
ko UMETA(DisplayName = "Korean"),
nl UMETA(DisplayName = "Dutch"),
pl UMETA(DisplayName = "Polish"),
ru UMETA(DisplayName = "Russian"),
sv UMETA(DisplayName = "Swedish"),
tr UMETA(DisplayName = "Turkish"),
tl UMETA(DisplayName = "Filipino"),
bg UMETA(DisplayName = "Bulgarian"),
ro UMETA(DisplayName = "Romanian"),
ar UMETA(DisplayName = "Arabic"),
cs UMETA(DisplayName = "Czech"),
el UMETA(DisplayName = "Greek"),
fi UMETA(DisplayName = "Finnish"),
hr UMETA(DisplayName = "Croatian"),
ms UMETA(DisplayName = "Malay"),
sk UMETA(DisplayName = "Slovak"),
da UMETA(DisplayName = "Danish"),
ta UMETA(DisplayName = "Tamil"),
uk UMETA(DisplayName = "Ukrainian"),
hu UMETA(DisplayName = "Hungarian"),
no UMETA(DisplayName = "Norwegian"),
vi UMETA(DisplayName = "Vietnamese"),
bn UMETA(DisplayName = "Bengali"),
th UMETA(DisplayName = "Thai"),
he UMETA(DisplayName = "Hebrew"),
ka UMETA(DisplayName = "Georgian"),
id UMETA(DisplayName = "Indonesian"),
te UMETA(DisplayName = "Telugu"),
gu UMETA(DisplayName = "Gujarati"),
kn UMETA(DisplayName = "Kannada"),
ml UMETA(DisplayName = "Malayalam"),
mr UMETA(DisplayName = "Marathi"),
pa UMETA(DisplayName = "Punjabi"),
};
UENUM(BlueprintType)
enum class EKeepAliveRule : uint8 {
EditorOnly UMETA(DisplayName = "Only keep Module open in Editor Mode"),
Never UMETA(DisplayName = "Never keep Module alive."),
Always UMETA(DisplayName = "Keep Module alive in Editor and Shipping mode.")
};

14
Unreal/Plugins/AvatarCore_Shared/Source/AvatarCore_Shared/Public/AvatarCore_Shared.h

@ -0,0 +1,14 @@
// Copyright Epic Games, Inc. All Rights Reserved.
#pragma once
#include "Modules/ModuleManager.h"
class FAvatarCore_SharedModule : public IModuleInterface
{
public:
/** IModuleInterface implementation */
virtual void StartupModule() override;
virtual void ShutdownModule() override;
};

27
Unreal/Plugins/AvatarCore_Shared/Source/AvatarCore_Shared/Public/FL_AvatarCoreShared.h

@ -0,0 +1,27 @@
// Fill out your copyright notice in the Description page of Project Settings.
#pragma once
#include "CoreMinimal.h"
#include "Kismet/BlueprintFunctionLibrary.h"
#include "AvatarCoreSharedEnums.h"
#include "FL_AvatarCoreShared.generated.h"
/**
*
*/
UCLASS()
class AVATARCORE_SHARED_API UFL_AvatarCoreShared : public UBlueprintFunctionLibrary
{
GENERATED_BODY()
public:
// Returns the ISO 639-1 code string for the given language enum (e.g. ELanguage::de "de").
// Derived from the enum value name via UE reflection, so it stays in sync automatically.
UFUNCTION(BlueprintCallable, Category = "AvatarCoreShared")
static FString LanguageToString(ELanguage Language);
// Returns the ELanguage for the given ISO 639-1 code string
UFUNCTION(BlueprintCallable, Category = "AvatarCoreShared")
static ELanguage StringToLanguage(const FString& InString);
};

4
Unreal/Plugins/AvatarCore_TTS/AvatarCore_TTS.uplugin

@ -25,6 +25,10 @@
{
"Name": "BTools",
"Enabled": true
},
{
"Name": "AvatarCore_Shared",
"Enabled": true
}
]
}

9
Unreal/Plugins/AvatarCore_TTS/Source/.claude/settings.local.json

@ -0,0 +1,9 @@
{
"permissions": {
"allow": [
"Bash(Get-ChildItem -Path \"c:\\\\Gitprojects\\\\SchaniConcierge\\\\Unreal\\\\Plugins\\\\AvatarCore_TTS\\\\Source\" -Recurse -Filter \"CartesiaTTSManager.h\")",
"Bash(Select-Object -ExpandProperty FullName)",
"Bash(find c:\\\\\\\\Gitprojects\\\\\\\\SchaniConcierge\\\\\\\\Unreal\\\\\\\\Plugins\\\\\\\\AvatarCore_TTS\\\\\\\\Source -name \"CartesiaTTSManager.h\")"
]
}
}

1
Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/AvatarCore_TTS.Build.cs

@ -35,6 +35,7 @@ public class AvatarCore_TTS : ModuleRules
"AudioMixer",
"AudioExtensions",
"WebRTC",
"AvatarCore_Shared",
// ... add other public dependencies that you statically link with here ...
}
);

62
Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Private/Cartesia/CartesiaTTSManager.cpp

@ -1,8 +1,8 @@
// Fill out your copyright notice in the Description page of Project Settings.
// Fill out your copyright notice in the Description page of Project Settings.
#include "Cartesia/CartesiaTTSManager.h"
#include "FL_AvatarCoreShared.h"
#include "WebSocketsModule.h"
#include "IWebSocket.h"
#include "Async/Async.h"
@ -168,7 +168,7 @@ void UCartesiaTTSManager::SendTranscriptMessage(int32 TaskID, const FString& Tra
TSharedPtr<FJsonObject> Obj = MakeShared<FJsonObject>();
Obj->SetStringField(TEXT("model_id"), CartesiaTTSConfig->CartesiaTTSSettings.CartesiaModelId);
Obj->SetStringField(TEXT("transcript"), Transcript);
Obj->SetStringField(TEXT("language"), TTSLanguageToString(CartesiaTTSConfig->GlobalTTSSettings.Language));
Obj->SetStringField(TEXT("language"), UFL_AvatarCoreShared::LanguageToString(CartesiaTTSConfig->GlobalTTSSettings.Language));
Obj->SetStringField(TEXT("context_id"), GetOrCreateContextId(TaskID));
Obj->SetBoolField(TEXT("continue"), bContinue);
@ -183,7 +183,7 @@ void UCartesiaTTSManager::SendTranscriptMessage(int32 TaskID, const FString& Tra
OutputObj->SetNumberField(TEXT("sample_rate"), (TTSConfig ? TTSConfig->GlobalTTSSettings.AudioSampleRate : 24000));
Obj->SetObjectField(TEXT("output_format"), OutputObj);
Obj->SetBoolField(TEXT("add_timestamps"), false);
Obj->SetBoolField(TEXT("add_timestamps"), true);
SendJsonForTask(TaskID, Obj);
@ -217,7 +217,7 @@ void UCartesiaTTSManager::SendFlushMessage(int32 TaskID)
TSharedPtr<FJsonObject> Obj = MakeShared<FJsonObject>();
Obj->SetStringField(TEXT("model_id"), CartesiaTTSConfig->CartesiaTTSSettings.CartesiaModelId);
Obj->SetStringField(TEXT("transcript"), TEXT(""));
Obj->SetStringField(TEXT("language"), TTSLanguageToString(CartesiaTTSConfig->GlobalTTSSettings.Language));
Obj->SetStringField(TEXT("language"), UFL_AvatarCoreShared::LanguageToString(CartesiaTTSConfig->GlobalTTSSettings.Language));
Obj->SetStringField(TEXT("context_id"), GetOrCreateContextId(TaskID));
Obj->SetBoolField(TEXT("continue"), true);
Obj->SetBoolField(TEXT("flush"), true);
@ -233,7 +233,7 @@ void UCartesiaTTSManager::SendFlushMessage(int32 TaskID)
OutputObj->SetNumberField(TEXT("sample_rate"), (TTSConfig ? TTSConfig->GlobalTTSSettings.AudioSampleRate : 24000));
Obj->SetObjectField(TEXT("output_format"), OutputObj);
Obj->SetBoolField(TEXT("add_timestamps"), false);
Obj->SetBoolField(TEXT("add_timestamps"), true);
SendJsonForTask(TaskID, Obj);
}
@ -330,6 +330,53 @@ void UCartesiaTTSManager::StartStreamingGeneration(int32 TaskID, const FString&
return;
}
if (Type.Equals(TEXT("timestamps"), ESearchCase::IgnoreCase))
{
const TSharedPtr<FJsonObject>* WTSObj;
if (Obj->TryGetObjectField(TEXT("word_timestamps"), WTSObj))
{
const TArray<TSharedPtr<FJsonValue>>* WordArr;
const TArray<TSharedPtr<FJsonValue>>* StartArr;
const TArray<TSharedPtr<FJsonValue>>* EndArr;
if ((*WTSObj)->TryGetArrayField(TEXT("words"), WordArr) &&
(*WTSObj)->TryGetArrayField(TEXT("start"), StartArr) &&
(*WTSObj)->TryGetArrayField(TEXT("end"), EndArr))
{
const int32 Count = FMath::Min3(WordArr->Num(), StartArr->Num(), EndArr->Num());
TArray<FWordTimestamp> Timestamps;
Timestamps.Reserve(Count);
for (int32 i = 0; i < Count; ++i)
{
FWordTimestamp WTS;
WTS.Word = (*WordArr)[i]->AsString();
WTS.Start = (float)(*StartArr)[i]->AsNumber();
WTS.End = (float)(*EndArr)[i]->AsNumber();
Timestamps.Add(WTS);
}
AsyncTask(ENamedThreads::GameThread, [Self, TaskID, TS = MoveTemp(Timestamps)]() mutable
{
FScopeLock Lock(&Self->TaskQueueCriticalSection);
for (FTTSTask& Task : Self->ActiveTasks)
{
if (Task.TaskID == TaskID)
{
// Deduplicate: only add words whose start time is after the last stored end time.
// Cartesia sends one timestamps message per audio chunk and may repeat words.
const float LastEnd = Task.WordTimestamps.IsEmpty() ? 0.f : Task.WordTimestamps.Last().End;
for (const FWordTimestamp& WTS : TS)
{
if (WTS.Start >= LastEnd - 0.01f)
Task.WordTimestamps.Add(WTS);
}
break;
}
}
});
}
}
return;
}
if (Type.Equals(TEXT("done"), ESearchCase::IgnoreCase))
{
bool bDone = false;
@ -353,8 +400,7 @@ void UCartesiaTTSManager::StartStreamingGeneration(int32 TaskID, const FString&
Self->TTSError(FString::Printf(TEXT("[Cartesia][%d] error: %s"), TaskID, *ErrorMsg));
Self->CloseAndRemoveSocket(TaskID, true);
return;
}
});
} });
/*WS->OnRawMessage().AddLambda([WeakThis, TaskID](const void* Data, SIZE_T Size, SIZE_T BytesRemaining)
{

90
Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Private/RealtimeAPI/RealtimeAPI_TTSManager.cpp

@ -8,3 +8,93 @@ void URealtimeAPI_TTSManager::InitTTSManager(UTTSBaseConfig* InTSSConfig, bool D
UTTSManagerBase::InitTTSManager(InTSSConfig, DebugMode);
SetTTSState(ETTSState::Ready);
}
void URealtimeAPI_TTSManager::AddAudioChunk(const TArray<uint8> AudioChunk, bool IsFinal)
{
// Flush any subtitle text that arrived before this first audio chunk created a task.
// Call the base implementation first so the task is created, then update OriginalText.
Super::AddAudioChunk(AudioChunk, IsFinal);
if (!PendingSubtitleText.IsEmpty())
{
FScopeLock Lock(&TaskQueueCriticalSection);
if (ActiveTasks.Num() > 0)
{
ActiveTasks.Last().OriginalText += PendingSubtitleText;
PendingSubtitleText.Empty();
}
}
}
void URealtimeAPI_TTSManager::AddTextChunk(const FString& TextChunk, bool IsFinal)
{
// Do not route through the base class: it would either create a TTS generation task
// (bSupportsStreamedInput=false path) or a competing streaming task (true path),
// both of which leave a phantom task in ActiveTasks that blocks playback forever.
bool bShouldSave = false;
bool bCacheHit = false;
FTTSTask TaskSnapshot;
{
FScopeLock Lock(&TaskQueueCriticalSection);
const FString FullText = PendingSubtitleText + TextChunk;
if (ActiveTasks.Num() > 0)
{
// AI audio path: a task already exists from AddAudioChunk.
// Update its subtitle text; handle cache save/load when text is final.
FTTSTask& Current = ActiveTasks.Last();
PendingSubtitleText.Empty();
Current.OriginalText += FullText;
if (IsFinal)
{
Current.Text = Current.OriginalText;
if (TTSConfig && TTSConfig->GlobalTTSSettings.UseCacheSystem)
{
if (!Current.bIsGenerating)
{
// Audio already done: save to cache now.
TaskSnapshot = Current;
bShouldSave = true;
}
// else: audio still arriving — OnGeneratedAudioChunkReceived saves
// once it sees Text is non-empty on the last audio chunk.
}
}
}
else if (IsFinal && TTSConfig && TTSConfig->GlobalTTSSettings.UseCacheSystem)
{
// Cache path: no AddAudioChunk was called. Create a task and try to load
// cached audio. If the cache misses, the task is discarded (caller should
// not reach here without cached audio available in pure-cache mode).
PendingSubtitleText.Empty();
FTTSTask NewTask;
NewTask.TaskID = NextTaskID++;
NewTask.Text = FullText;
NewTask.OriginalText = FullText;
NewTask.bIsGenerating = true;
if (TryLoadCachedAudioIntoTask(NewTask))
{
NewTask.bIsGenerating = false;
NewTask.AudioStartByteOffset = 0;
ActiveTasks.Add(NewTask);
bReceivedFinalInput = true;
bCacheHit = true;
}
// Cache miss in pure-cache mode: nothing to play, task discarded.
}
else
{
// Audio task not created yet; buffer until AddAudioChunk creates it.
PendingSubtitleText += TextChunk;
}
}
if (bShouldSave)
SaveTaskAudioToCache(TaskSnapshot);
if (bCacheHit)
CheckPlayback();
}

331
Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Private/TTSManagerBase.cpp

@ -5,8 +5,14 @@
#include "AvatarSoundWave.h"
#include "Misc/App.h"
#include "Misc/SecureHash.h"
#include "FL_AvatarCoreShared.h"
#include "AvatarCoreSharedEnums.h"
#include "Kismet/GameplayStatics.h"
#include "HAL/FileManager.h"
#include "Dom/JsonObject.h"
#include "Serialization/JsonSerializer.h"
#include "Serialization/JsonWriter.h"
#include "Serialization/JsonReader.h"
UTTSManagerBase::UTTSManagerBase()
: RegisteredAudioComponent(nullptr)
@ -294,6 +300,12 @@ void UTTSManagerBase::WipeLastCachedFile()
{
IFileManager::Get().Delete(*NewestFullPath, /*RequireExists=*/false, /*EvenReadOnly=*/true, /*Quiet=*/true);
UE_LOG(LogTemp, Warning, TEXT("File deleted: %s"), *NewestFullPath);
const FString SidecarPath = FPaths::ChangeExtension(NewestFullPath, TEXT("json"));
if (FPaths::FileExists(SidecarPath))
{
IFileManager::Get().Delete(*SidecarPath, /*RequireExists=*/false, /*EvenReadOnly=*/true, /*Quiet=*/true);
}
}
}
@ -306,6 +318,9 @@ void UTTSManagerBase::OnTTSManagerFinished()
ProceduralSoundWave->Duration = 0.0f;
}
AccumulatedSubtitleText.Empty();
StreamingInputBuffer.Empty();
SetTTSState(ETTSState::Ready);
}
@ -365,6 +380,8 @@ void UTTSManagerBase::ClearTTS()
// Reset state variables
bReceivedFinalInput = false;
AccumulatedSubtitleText.Empty();
StreamingInputBuffer.Empty();
// Set state to Ready
SetTTSState(ETTSState::Ready);
@ -378,7 +395,7 @@ void UTTSManagerBase::SetCachingEnabled(bool bEnabled)
TTSConfig->GlobalTTSSettings.UseCacheSystem = bEnabled;
}
void UTTSManagerBase::SetLanguage(ETTSLanguage NewLanguage)
void UTTSManagerBase::SetTTSLanguage(ELanguage NewLanguage)
{
if (!TTSConfig) return;
TTSConfig->GlobalTTSSettings.Language = NewLanguage;
@ -416,6 +433,59 @@ void UTTSManagerBase::AddTextChunk(const FString& TextChunk, bool IsFinal)
if (bShouldStream)
{
// Accumulate into buffer and flush only at word boundaries to allow multi-word replacements
StreamingInputBuffer += TextChunk;
FString ToSend;
if (IsFinal)
{
ToSend = StreamingInputBuffer;
StreamingInputBuffer.Empty();
}
else
{
// Find last word boundary scanning from the end
int32 LastBoundary = -1;
for (int32 i = StreamingInputBuffer.Len() - 1; i >= 0; --i)
{
TCHAR C = StreamingInputBuffer[i];
if (C == ' ' || C == '.' || C == ',' || C == '!' || C == '?' || C == ':' || C == ';')
{
LastBoundary = i;
break;
}
}
if (LastBoundary >= 0)
{
ToSend = StreamingInputBuffer.Left(LastBoundary + 1);
StreamingInputBuffer = StreamingInputBuffer.Mid(LastBoundary + 1);
}
}
// Nothing ready yet: still accumulating mid-word
if (ToSend.IsEmpty())
{
if (!IsFinal) return;
if (!bHasActiveStreamingTask) return;
// IsFinal + active task with empty ToSend: fall through to close the task
}
// Apply word replacements to the flushed portion
FString ProcessedChunk = ToSend;
if (TTSConfig && TTSConfig->GlobalTTSSettings.WordReplacements.Num() > 0)
{
for (const FString& Entry : TTSConfig->GlobalTTSSettings.WordReplacements)
{
FString From, To;
if (Entry.Split(TEXT("|"), &From, &To))
{
From = From.TrimStartAndEnd();
To = To.TrimStartAndEnd();
if (!From.IsEmpty())
ProcessedChunk.ReplaceInline(*From, *To, ESearchCase::IgnoreCase);
}
}
}
bool bStartedNewStreamingTask = false;
@ -429,8 +499,8 @@ void UTTSManagerBase::AddTextChunk(const FString& TextChunk, bool IsFinal)
{
FTTSTask NewTask;
NewTask.TaskID = NextTaskID++;
NewTask.OriginalText = TextChunk;
NewTask.Text = TextChunk;
NewTask.OriginalText = ToSend;
NewTask.Text = ProcessedChunk;
NewTask.bIsStreaming = !IsFinal;
NewTask.bIsGenerating = true;
ActiveTasks.Add(NewTask);
@ -440,11 +510,19 @@ void UTTSManagerBase::AddTextChunk(const FString& TextChunk, bool IsFinal)
}
else
{
CurrentTask.OriginalText += TextChunk;
CurrentTask.Text += TextChunk;
// CurrentTask is a local copy; update the actual element in ActiveTasks
for (FTTSTask& ActiveTask : ActiveTasks)
{
if (ActiveTask.TaskID == StreamingTaskID)
{
ActiveTask.OriginalText += ToSend;
ActiveTask.Text += ProcessedChunk;
if (IsFinal)
{
CurrentTask.bIsStreaming = false;
ActiveTask.bIsStreaming = false;
}
break;
}
}
}
@ -453,11 +531,11 @@ void UTTSManagerBase::AddTextChunk(const FString& TextChunk, bool IsFinal)
if (bStartedNewStreamingTask)
{
SetTTSState(ETTSState::Producing);
StartStreamingGeneration(StreamingTaskID, TextChunk, IsFinal);
StartStreamingGeneration(StreamingTaskID, ProcessedChunk, IsFinal);
}
else
{
UpdateStreamingText(StreamingTaskID, TextChunk, IsFinal);
UpdateStreamingText(StreamingTaskID, ProcessedChunk, IsFinal);
}
return;
}
@ -547,16 +625,6 @@ void UTTSManagerBase::OnGeneratedAudioChunkReceived(FTTSTask& Task, const TArray
ProcessedAudio = AudioData;
}
// Run through WebRTC APM for chunk-boundary smoothing (NS + HPF)
if (WebRTCChannel && WebRTCChannel->IsInitialized() && ProcessedAudio.Num() > 0)
{
TArray<uint8> WebRTCProcessed = WebRTCChannel->ProcessTTSAudio(ProcessedAudio);
if (WebRTCProcessed.Num() > 0)
{
ProcessedAudio = MoveTemp(WebRTCProcessed);
}
}
// Append audio data to the owning ActiveTask buffer, not directly to the output queue
FTTSTask* TargetTaskPtr = nullptr;
{
@ -577,6 +645,12 @@ void UTTSManagerBase::OnGeneratedAudioChunkReceived(FTTSTask& Task, const TArray
return;
}
// Record where speech audio starts (before the first real chunk is appended)
if (TargetTaskPtr->AudioStartByteOffset < 0 && ProcessedAudio.Num() > 0)
{
TargetTaskPtr->AudioStartByteOffset = TargetTaskPtr->AudioData.Num();
}
// Append first
TargetTaskPtr->AudioData.Append(ProcessedAudio);
@ -604,9 +678,16 @@ void UTTSManagerBase::OnGeneratedAudioChunkReceived(FTTSTask& Task, const TArray
TargetTaskPtr->bIsComplete = true;
}
}
if (IsLastChunk && TTSConfig->GlobalTTSSettings.UseCacheSystem && !bIsPreCaching)
if (IsLastChunk && TTSConfig->GlobalTTSSettings.UseCacheSystem && !bIsPreCaching && TargetTaskPtr && !TargetTaskPtr->Text.IsEmpty())
{
SaveTaskAudioToCache(Task);
// Use TargetTaskPtr (actual task) not Task (may be a stale local copy from AddAudioChunk).
// TargetTaskPtr has the complete AudioData including the last chunk just appended above,
// and the correct Text field. Access outside the lock is safe: bIsGenerating=false means
// no further audio appends can occur on this task.
// Note: for RealtimeAPI, Text may still be empty here if the final text chunk has not
// arrived yet — in that case, RealtimeAPI_TTSManager::AddTextChunk triggers the save
// once both streams are finalised.
SaveTaskAudioToCache(*TargetTaskPtr);
}
UTTSManagerBase::CheckPlayback();
}
@ -739,6 +820,22 @@ void UTTSManagerBase::OnAvatarSoundwaveBufferUnderun()
SetTTSState(ETTSState::WaitingForChunks);
TTSLog(TEXT("Stopping AudioComponent playback"));
RegisteredAudioComponent->Stop();
// Stop() wipes the ProceduralSoundWave internal buffer. Reset QueueCursor to
// PlaybackCursor so those bytes can be re-queued by FlushToAudioQueue.
// Clear QueuedSegments since the audio device will no longer render those bytes.
{
FScopeLock TaskLock(&TaskQueueCriticalSection);
for (FTTSTask& Task : ActiveTasks)
{
Task.QueueCursor = Task.PlaybackCursor;
}
}
{
FScopeLock SegLock(&SegmentQueueCriticalSection);
QueuedSegments.Empty();
}
if (WebRTCChannel) WebRTCChannel->Reset();
}
}
@ -909,18 +1006,42 @@ bool UTTSManagerBase::FlushToAudioQueue()
if (ProceduralSoundWave)
{
TTSLog(FString::Printf(TEXT("Queueing %d bytes for task %d"), BytesToQueue, Curr.TaskID));
// Queue the available tail for this task
ProceduralSoundWave->QueueAudio(Curr.AudioData.GetData() + BytesQueuedSoFar, BytesToQueue);
// Apply WebRTC HPF smoothing for live playback; Task.AudioData stays raw for clean caching
TArray<uint8> AudioToQueue(Curr.AudioData.GetData() + BytesQueuedSoFar, BytesToQueue);
if (WebRTCChannel && WebRTCChannel->IsInitialized())
{
TArray<uint8> Processed = WebRTCChannel->ProcessTTSAudio(AudioToQueue);
if (Processed.Num() > 0)
AudioToQueue = MoveTemp(Processed);
}
ProceduralSoundWave->QueueAudio(AudioToQueue.GetData(), AudioToQueue.Num());
// Update duration accurately: seconds = bytes / (bytesPerSample * channels * sampleRate)
const float DurationInc = static_cast<float>(BytesToQueue) / static_cast<float>(BytesPerSecond);
const float DurationInc = static_cast<float>(AudioToQueue.Num()) / static_cast<float>(BytesPerSecond);
ProceduralSoundWave->Duration += DurationInc;
// Drain WebRTC accumulator on the final segment so no audio is held back.
// Use !bIsGenerating (not bIsComplete) — bIsComplete requires PlaybackCursor to catch up,
// which hasn't happened yet. bIsGenerating=false means the API sent all chunks.
if (!Curr.bIsGenerating && (Curr.QueueCursor + BytesToQueue >= Curr.AudioData.Num()))
{
if (WebRTCChannel && WebRTCChannel->IsInitialized())
{
TArray<uint8> Tail = WebRTCChannel->Flush();
if (Tail.Num() > 0)
{
ProceduralSoundWave->QueueAudio(Tail.GetData(), Tail.Num());
ProceduralSoundWave->Duration += static_cast<float>(Tail.Num()) / static_cast<float>(BytesPerSecond);
}
}
}
}
// Advance queue cursor but DO NOT mark as rendered yet
// Capture offset before advancing the cursor so ConsumeRenderedBytes can map bytes→time
const int32 SegmentStartOffset = Curr.QueueCursor;
// Advance by AudioData bytes (keeps PlaybackCursor in AudioData coordinate space for subtitle timing)
Curr.QueueCursor += BytesToQueue;
// Track this queued segment for precise playback accounting
{
FScopeLock SegLock(&SegmentQueueCriticalSection);
QueuedSegments.Add({ Curr.TaskID, BytesToQueue });
QueuedSegments.Add({ Curr.TaskID, BytesToQueue, SegmentStartOffset });
}
bQueuedAny = true;
@ -1103,8 +1224,7 @@ void UTTSManagerBase::ProcessText(bool IsFinal)
}
if (IsFinal && !Text.IsEmpty())
{
FString Sentence = Text;
Sentence = ProcessTextForGeneration(Sentence);
const FString Sentence = Text;
if (!Sentence.IsEmpty())
{
// Directly add to PendingTasks queue instead of redundant SentenceQueue
@ -1280,10 +1400,21 @@ FString UTTSManagerBase::GetCacheDirectory() const
FString UTTSManagerBase::GetCacheFilePath(const FString& InText) const
{
const FString Hash = ComputeTaskHash(InText);
UE_LOG(LogTemp, Warning, TEXT("InText is empty therefore hash cannot be created, using invalidCacheNoText instead."));
FString Hash;
if (InText.IsEmpty())
Hash = "invalidCacheNoText";
else
Hash = ComputeTaskHash(InText);
return FPaths::Combine(GetCacheDirectory(), Hash + TEXT(".wav"));
}
FString UTTSManagerBase::GetCacheSidecarPath(const FString& InText) const
{
const FString Hash = ComputeTaskHash(InText);
return FPaths::Combine(GetCacheDirectory(), Hash + TEXT(".json"));
}
bool UTTSManagerBase::HasCacheFile(const FString& InText)
{
const FString FilePath = GetCacheFilePath(ProcessTextForGeneration(InText));
@ -1380,6 +1511,32 @@ bool UTTSManagerBase::SaveTaskAudioToCache(const FTTSTask& Task)
if (bOk)
{
TTSLog(FString::Printf(TEXT("Saved cache: %s (%d bytes)"), *FilePath, Wav.Num()));
if (Task.WordTimestamps.Num() > 0)
{
TSharedRef<FJsonObject> Root = MakeShared<FJsonObject>();
TArray<TSharedPtr<FJsonValue>> WordsArray;
for (const FWordTimestamp& WT : Task.WordTimestamps)
{
TSharedRef<FJsonObject> Entry = MakeShared<FJsonObject>();
Entry->SetStringField(TEXT("word"), WT.Word);
Entry->SetNumberField(TEXT("start"), WT.Start);
Entry->SetNumberField(TEXT("end"), WT.End);
WordsArray.Add(MakeShared<FJsonValueObject>(Entry));
}
Root->SetArrayField(TEXT("words"), WordsArray);
FString JsonStr;
TSharedRef<TJsonWriter<>> Writer = TJsonWriterFactory<>::Create(&JsonStr);
FJsonSerializer::Serialize(Root, Writer);
const FString SidecarPath = GetCacheSidecarPath(Task.Text);
if (!FFileHelper::SaveStringToFile(JsonStr, *SidecarPath))
{
TTSLog(FString::Printf(TEXT("Failed to save timestamp sidecar: %s"), *SidecarPath));
}
}
}
else
{
@ -1413,6 +1570,37 @@ bool UTTSManagerBase::TryLoadCachedAudioIntoTask(FTTSTask& Task)
Task.AudioData.Append(DataPtr, DataSize);
IFileManager::Get().SetTimeStamp(*FilePath, FDateTime::Now());
const FString SidecarPath = GetCacheSidecarPath(Task.Text);
if (FPaths::FileExists(SidecarPath))
{
FString JsonStr;
if (FFileHelper::LoadFileToString(JsonStr, *SidecarPath))
{
TSharedPtr<FJsonObject> Root;
TSharedRef<TJsonReader<>> Reader = TJsonReaderFactory<>::Create(JsonStr);
if (FJsonSerializer::Deserialize(Reader, Root) && Root.IsValid())
{
const TArray<TSharedPtr<FJsonValue>>* WordsArr;
if (Root->TryGetArrayField(TEXT("words"), WordsArr))
{
for (const TSharedPtr<FJsonValue>& Val : *WordsArr)
{
const TSharedPtr<FJsonObject>* Obj;
if (Val->TryGetObject(Obj))
{
FWordTimestamp WT;
(*Obj)->TryGetStringField(TEXT("word"), WT.Word);
(*Obj)->TryGetNumberField(TEXT("start"), WT.Start);
(*Obj)->TryGetNumberField(TEXT("end"), WT.End);
Task.WordTimestamps.Add(WT);
}
}
}
}
}
}
return true;
}
@ -1421,6 +1609,15 @@ void UTTSManagerBase::ConsumeRenderedBytes(int32 Bytes)
if (Bytes <= 0)
return;
if (!TTSConfig)
return;
const int32 BytesPerSample = 2; // 16-bit PCM
const int32 EffSR = (TTSConfig->GlobalTTSSettings.ResampleToSampleRate > 0)
? TTSConfig->GlobalTTSSettings.ResampleToSampleRate
: TTSConfig->GlobalTTSSettings.AudioSampleRate;
const int32 BytesPerSecond = EffSR * TTSConfig->GlobalTTSSettings.AudioNumChannels * BytesPerSample;
// Drain the queued segment FIFO while updating per-task playback
// Lock order: SegmentQueue first, then TaskQueue to avoid deadlock with FlushToAudioQueue
FScopeLock SegLock(&SegmentQueueCriticalSection);
@ -1450,6 +1647,84 @@ void UTTSManagerBase::ConsumeRenderedBytes(int32 Bytes)
{
Task.bIsComplete = true;
}
// --- Subtitle event ---
TArray<FString> OrigWords;
Task.OriginalText.ParseIntoArray(OrigWords, TEXT(" "), true);
if (OrigWords.Num() > 0)
{
int32 NewWordIdx = Task.LastSubtitleWordIndex;
// TimeNow is relative to the start of Cartesia's audio (not the task buffer start),
// so subtract any silence that was prepended before the first real chunk.
const int32 SpeechStart = (Task.AudioStartByteOffset >= 0) ? Task.AudioStartByteOffset : 0;
const int32 SpeechCursor = FMath::Max(Task.PlaybackCursor - SpeechStart, 0);
const int32 SpeechTotal = FMath::Max(Task.AudioData.Num() - SpeechStart, 0);
if (Task.WordTimestamps.Num() > 0 && BytesPerSecond > 0)
{
// Timestamp path: advance at most one word per callback so timestamps that
// arrived before audio started playing don't all fire in one burst.
const float TimeNow = (float)SpeechCursor / (float)BytesPerSecond;
const int32 NextWi = Task.LastSubtitleWordIndex + 1;
if (NextWi < Task.WordTimestamps.Num() && Task.WordTimestamps[NextWi].Start <= TimeNow)
{
NewWordIdx = NextWi;
}
}
else
{
// Fallback: proportional interpolation, also one word at a time
const float Fraction = (SpeechTotal > 0)
? FMath::Clamp((float)SpeechCursor / (float)SpeechTotal, 0.f, 1.f)
: 0.f;
const int32 ProportionalIdx = FMath::Clamp(
FMath::FloorToInt(Fraction * OrigWords.Num()) - 1,
Task.LastSubtitleWordIndex, OrigWords.Num() - 1);
if (ProportionalIdx > Task.LastSubtitleWordIndex)
NewWordIdx = Task.LastSubtitleWordIndex + 1;
}
if (NewWordIdx > Task.LastSubtitleWordIndex)
{
FString ChunkText;
for (int32 wi = Task.LastSubtitleWordIndex + 1; wi <= NewWordIdx; ++wi)
{
if (OrigWords.IsValidIndex(wi))
{
if (!ChunkText.IsEmpty()) ChunkText += TEXT(" ");
ChunkText += OrigWords[wi];
}
}
Task.LastSubtitleWordIndex = NewWordIdx;
if (!ChunkText.IsEmpty())
{
AccumulatedSubtitleText += (AccumulatedSubtitleText.IsEmpty() ? TEXT("") : TEXT(" ")) + ChunkText;
}
const bool bIsFinal = Task.bIsComplete
&& Task.PlaybackCursor >= Task.AudioData.Num()
&& QueuedSegments.Num() <= 1;
if (!ChunkText.IsEmpty() || bIsFinal)
{
const int32 FireTaskID = Task.TaskID;
FString ChunkCopy = ChunkText;
FString AccumCopy = AccumulatedSubtitleText;
TWeakObjectPtr<UTTSManagerBase> WeakThis(this);
AsyncTask(ENamedThreads::GameThread, [WeakThis, FireTaskID, ChunkCopy, AccumCopy, bIsFinal]()
{
if (WeakThis.IsValid())
{
WeakThis->OnTTSSubtitle.Broadcast(FireTaskID, ChunkCopy, AccumCopy, bIsFinal);
}
});
}
}
}
// --- End subtitle event ---
break;
}
}

35
Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Private/TTSWebRTCChannel.cpp

@ -53,9 +53,8 @@ void FTTSWebRTCChannel::Configure()
Config.echo_canceller.enabled = false; // AEC handled by AvatarCore_STT
Config.pre_amplifier.enabled = false;
Config.high_pass_filter.enabled = true; // removes DC offset / low-freq rumble
Config.noise_suppression.enabled = true; // smooths inter-chunk artifacts
Config.noise_suppression.level = webrtc::AudioProcessing::Config::NoiseSuppression::kLow;
Config.transient_suppression.enabled = true; // suppresses click artifacts at chunk boundaries
Config.noise_suppression.enabled = false; // causes artifacts on clean TTS audio
Config.transient_suppression.enabled = false; // designed for mic capture, not synthesis
Config.gain_controller1.enabled = false;
Config.gain_controller2.enabled = false;
@ -79,6 +78,36 @@ void FTTSWebRTCChannel::Reset()
}
}
// ---------------------------------------------------------------------------
TArray<uint8> FTTSWebRTCChannel::Flush()
{
FScopeLock Lock(&CriticalSection);
if (Accumulator.Num() == 0 || !bInitialized || !AudioProcessing || FrameSamplesTotal <= 0)
return {};
// Zero-pad to a complete frame
while (Accumulator.Num() < FrameSamplesTotal)
Accumulator.Add(0);
webrtc::StreamConfig StreamCfg(WebRTCRate, NumChannels);
TArray<int16> FrameBuf;
FrameBuf.SetNumUninitialized(FrameSamplesTotal);
FMemory::Memcpy(FrameBuf.GetData(), Accumulator.GetData(), FrameSamplesTotal * sizeof(int16));
AudioProcessing->ProcessStream(FrameBuf.GetData(), StreamCfg, StreamCfg, FrameBuf.GetData());
Accumulator.Reset();
TArray<int16> FinalInt16;
if (SampleRate != WebRTCRate)
ResampleInt16Linear(FrameBuf, WebRTCRate, SampleRate, FinalInt16);
else
FinalInt16 = MoveTemp(FrameBuf);
TArray<uint8> Out;
Out.SetNumUninitialized(FinalInt16.Num() * 2);
FMemory::Memcpy(Out.GetData(), FinalInt16.GetData(), Out.Num());
return Out;
}
// ---------------------------------------------------------------------------
TArray<uint8> FTTSWebRTCChannel::ProcessTTSAudio(const TArray<uint8>& InPCM16)
{

9
Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Public/RealtimeAPI/RealtimeAPI_TTSManager.h

@ -18,4 +18,13 @@ public:
// UTTSManagerBase overrides
virtual void InitTTSManager(UTTSBaseConfig* InTSSConfig, bool DebugMode) override;
// In the RealtimeAPI flow, audio comes directly via AddAudioChunk while text is piped
// in separately for subtitle display. These overrides keep text and audio on the same
// task without creating additional TTS generation tasks.
virtual void AddTextChunk(const FString& TextChunk, bool IsFinal) override;
virtual void AddAudioChunk(const TArray<uint8> AudioChunk, bool IsFinal) override;
private:
// Buffer for text that arrives before the first audio chunk creates a task
FString PendingSubtitleText;
};

73
Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Public/TTSBaseConfig.h

@ -4,6 +4,7 @@
#include "CoreMinimal.h"
#include "UObject/NoExportTypes.h"
#include "AvatarCoreSharedEnums.h"
#include "TTSBaseConfig.generated.h"
UENUM(BlueprintType)
@ -14,76 +15,6 @@ enum class ECommaSplitRule : uint8 {
FillWord UMETA(DisplayName = "Split on every comma, but add fill words.")
};
UENUM(BlueprintType)
enum class ETTSKeepAliveRule : uint8 {
EditorOnly UMETA(DisplayName = "Only keep TTS Module open in Editor Mode"),
Never UMETA(DisplayName = "Never keep TTS Module alive."),
Always UMETA(DisplayName = "Keep TTS Module alive in Editor and Shipping mode.")
};
UENUM(BlueprintType)
enum class ETTSLanguage : uint8
{
NONE UMETA(DisplayName = "Unset"),
en UMETA(DisplayName = "English"),
fr UMETA(DisplayName = "French"),
de UMETA(DisplayName = "German"),
es UMETA(DisplayName = "Spanish"),
pt UMETA(DisplayName = "Portuguese"),
zh UMETA(DisplayName = "Chinese"),
ja UMETA(DisplayName = "Japanese"),
hi UMETA(DisplayName = "Hindi"),
it UMETA(DisplayName = "Italian"),
ko UMETA(DisplayName = "Korean"),
nl UMETA(DisplayName = "Dutch"),
pl UMETA(DisplayName = "Polish"),
ru UMETA(DisplayName = "Russian"),
sv UMETA(DisplayName = "Swedish"),
tr UMETA(DisplayName = "Turkish"),
tl UMETA(DisplayName = "Filipino"),
bg UMETA(DisplayName = "Bulgarian"),
ro UMETA(DisplayName = "Romanian"),
ar UMETA(DisplayName = "Arabic"),
cs UMETA(DisplayName = "Czech"),
el UMETA(DisplayName = "Greek"),
fi UMETA(DisplayName = "Finnish"),
hr UMETA(DisplayName = "Croatian"),
ms UMETA(DisplayName = "Malay"),
sk UMETA(DisplayName = "Slovak"),
da UMETA(DisplayName = "Danish"),
ta UMETA(DisplayName = "Tamil"),
uk UMETA(DisplayName = "Ukrainian"),
hu UMETA(DisplayName = "Hungarian"),
no UMETA(DisplayName = "Norwegian"),
vi UMETA(DisplayName = "Vietnamese"),
bn UMETA(DisplayName = "Bengali"),
th UMETA(DisplayName = "Thai"),
he UMETA(DisplayName = "Hebrew"),
ka UMETA(DisplayName = "Georgian"),
id UMETA(DisplayName = "Indonesian"),
te UMETA(DisplayName = "Telugu"),
gu UMETA(DisplayName = "Gujarati"),
kn UMETA(DisplayName = "Kannada"),
ml UMETA(DisplayName = "Malayalam"),
mr UMETA(DisplayName = "Marathi"),
pa UMETA(DisplayName = "Punjabi"),
};
// Returns the ISO 639-1 code string for the given language enum (e.g. ETTSLanguage::de → "de").
// Derived from the enum value name via UE reflection, so it stays in sync automatically.
static FORCEINLINE FString TTSLanguageToString(ETTSLanguage Language)
{
FString Name = StaticEnum<ETTSLanguage>()->GetNameStringByValue(static_cast<int64>(Language));
// StaticEnum returns the short name (no "ETTSLanguage::" prefix in UE5 GetNameStringByValue)
// but strip the prefix defensively in case the runtime returns the qualified form.
int32 ScopePos;
if (Name.FindLastChar(TEXT(':'), ScopePos))
{
Name = Name.RightChop(ScopePos + 1);
}
return Name.IsEmpty() ? TEXT("en") : Name;
}
USTRUCT(BlueprintType)
struct FGlobalTTSSettings
{
@ -134,7 +65,7 @@ struct FGlobalTTSSettings
int32 MaxCharacterForGeneration = 0;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreTTS|Base", meta = (ExposeOnSpawn = "true"))
ETTSLanguage Language = ETTSLanguage::de;
ELanguage Language = ELanguage::de;
};
class UTTSManagerBase;

36
Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Public/TTSManagerBase.h

@ -29,6 +29,15 @@ enum class ETTSState : uint8
FinishPlayback UMETA(DisplayName = "Finish Playback")
};
USTRUCT()
struct FWordTimestamp
{
GENERATED_BODY()
FString Word;
float Start = 0.f;
float End = 0.f;
};
// TTS Task struct for concurrent generation
USTRUCT(BlueprintType)
struct FTTSTask
@ -48,6 +57,9 @@ struct FTTSTask
// Queued cursor: how many bytes have been queued to the procedural wave (may be ahead of PlaybackCursor)
int32 QueueCursor = 0;
bool bIsComplete = false; // True when all chunks are received and played
TArray<FWordTimestamp> WordTimestamps; // populated async by Cartesia timestamp messages
int32 LastSubtitleWordIndex = -1; // last word index fired in TTSSubtitle event
int32 AudioStartByteOffset = -1; // byte offset in AudioData where first real audio chunk begins (after any prepended silence)
FTTSTask() : Text(), OriginalText(), TaskID(0) {}
FTTSTask(const FString& InText, FString& InOriginalText, int32 InTaskID) : Text(InText), OriginalText(InOriginalText), TaskID(InTaskID) {}
};
@ -58,6 +70,7 @@ DECLARE_DYNAMIC_MULTICAST_DELEGATE(FOnTTSCacheFinished);
DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FOnTTSError, const FString&, ErrorMessage);
DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FOnTTSStateChanged, ETTSState, NewState);
DECLARE_DYNAMIC_MULTICAST_DELEGATE_ThreeParams(FOnTTSAudioChunkForBP, const TArray<float>&, PCMData, int32, SampleRate, int32, NumChannels);
DECLARE_DYNAMIC_MULTICAST_DELEGATE_FourParams(FOnTTSSubtitle, int32, TaskID, const FString&, CurrentChunkText, const FString&, AccumulatedSubtitle, bool, bIsFinal);
UCLASS(Abstract, Blueprintable, BlueprintType)
class AVATARCORE_TTS_API UTTSManagerBase : public UObject
@ -92,6 +105,10 @@ public:
UPROPERTY(BlueprintAssignable, Category = "AvatarCoreTTS|Events")
FOnTTSCacheUpdate TTSCacheUpdate;
// Fired as words are spoken; CurrentChunkText = words spoken since last event, AccumulatedSubtitle = all words spoken so far
UPROPERTY(BlueprintAssignable, Category = "AvatarCoreTTS|Events")
FOnTTSSubtitle OnTTSSubtitle;
// Initializes the Manager and set Config file
UFUNCTION(BlueprintCallable, Category = "AvatarCoreTTS|Lifecycle")
virtual void InitTTSManager(UTTSBaseConfig* InTSSConfig, bool DebugMode);
@ -109,7 +126,7 @@ public:
// Set the language used for TTS generation
UFUNCTION(BlueprintCallable, Category = "AvatarCoreTTS|Settings")
void SetLanguage(ETTSLanguage NewLanguage);
void SetTTSLanguage(ELanguage NewLanguage);
// Is the TTSManager currently busy
UFUNCTION(BlueprintCallable, Category = "AvatarCoreTTS|Helper")
@ -217,8 +234,6 @@ protected:
void SetTTSState(ETTSState NewState);
void AddEndSilence(FTTSTask& Task);
// Sentence segmentation
FString PendingText;
FCriticalSection PendingTextCriticalSection;
@ -244,9 +259,6 @@ protected:
virtual void AddToRootManager();
virtual void RemoveFromRootManager();
// Centralized Audio Queue Management
void AddSilenceToTask(FTTSTask& Task, float SilenceLength);
bool bIsRooted;
// Timers
@ -281,6 +293,9 @@ private:
TArray<FString> PreCacheSentences;
// Accumulates streaming text until a word boundary before applying replacements and forwarding
FString StreamingInputBuffer;
TUniquePtr<FTTSWebRTCChannel> WebRTCChannel;
// Timer for polling Ready transition after playback finished
@ -294,17 +309,20 @@ protected:
FString GetCacheDirectory() const;
// Full cache file path for given text (Hash.wav)
FString GetCacheFilePath(const FString& InText) const;
// Sidecar JSON path for word timestamps (Hash.json, same stem as wav)
FString GetCacheSidecarPath(const FString& InText) const;
// Try to load cached WAV into Task.AudioData (returns true on success)
bool TryLoadCachedAudioIntoTask(FTTSTask& Task);
// Save Task.AudioData as WAV to cache (returns true on success)
bool SaveTaskAudioToCache(const FTTSTask& Task);
bool bInitialSilenceDone = false;
void OnGeneratedAudioChunkReceived(FTTSTask& Task, const TArray<uint8>& AudioData, bool IsLastChunk);
// Accumulated subtitle text across the current speech session; cleared by ClearTTS
FString AccumulatedSubtitleText;
// Accurate playback accounting without relying on underflow: FIFO of queued segments
struct FQueuedSegment { int32 TaskID; int32 BytesRemaining; };
struct FQueuedSegment { int32 TaskID; int32 BytesRemaining; int32 StartByteOffset; };
FCriticalSection SegmentQueueCriticalSection;
TArray<FQueuedSegment> QueuedSegments; // treated as a queue (pop from front)

3
Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Public/TTSWebRTCChannel.h

@ -41,6 +41,9 @@ public:
/** Flush and discard any accumulated partial frame. */
void Reset();
/** Zero-pad accumulator to a full frame, process it, return the result, then clear the accumulator. Returns empty if accumulator is empty. */
TArray<uint8> Flush();
bool IsInitialized() const { return bInitialized; }
private:

2
Unreal/Plugins/BLogger/Source/BLogger/Public/SLogWidgetState.h

@ -15,7 +15,7 @@ private:
bool IsScrollToBottomEnabled = true;
bool IsAutoRefreshEnabled = true;
TArray<FString> FilteredCategories;
int32 VerbosityFilterMask = 127;
int32 VerbosityFilterMask = 126; // VeryVerbose (1) excluded by default
FString SearchText;
public:

BIN
Unreal/Plugins/BTools/Content/Components/CheckProcessComponent.uasset (Stored with Git LFS)

Binary file not shown.

30
Unreal/Plugins/BTools/Source/BTools/Private/BToolsBPLibrary.cpp

@ -119,8 +119,12 @@ bool UBToolsBPLibrary::KillProcessByName(FString Name)
#endif
}
int UBToolsBPLibrary::KillProcessesByPath(FString Dir)
int UBToolsBPLibrary::KillProcessesByPath(FString Dir, bool UseAbsolutePath)
{
if (Dir.IsEmpty())
return 0;
#if PLATFORM_WINDOWS
int32 ReturnCode = 0; FString Out, Err;
int32 Killed = 0;
@ -141,8 +145,6 @@ int UBToolsBPLibrary::KillProcessesByPath(FString Dir)
{
do
{
if (FCString::Stricmp(TEXT("python.exe"), Entry.szExeFile) == 0)
{
HANDLE ProcessHandle = ::OpenProcess(PROCESS_TERMINATE | PROCESS_QUERY_LIMITED_INFORMATION, /*bInherit*/ false, Entry.th32ProcessID);
if (ProcessHandle)
@ -153,14 +155,25 @@ int UBToolsBPLibrary::KillProcessesByPath(FString Dir)
FString ProcPath(Buffer);
ProcPath.ReplaceInline(TEXT("/"), TEXT("\\"));
// Case-insensitive prefix check
if (UseAbsolutePath)
{
if (ProcPath.Len() >= ScriptsDirFull.Len() && ProcPath.Left(ScriptsDirFull.Len()).Compare(ScriptsDirFull, ESearchCase::IgnoreCase) == 0)
{
::TerminateProcess(ProcessHandle, 0);
++Killed;
}
}
::CloseHandle(ProcessHandle);
else //Only contains string
{
if (ProcPath.Contains(Dir, ESearchCase::Type::IgnoreCase, ESearchDir::FromStart))
{
::TerminateProcess(ProcessHandle, 0);
++Killed;
}
}
}
::CloseHandle(ProcessHandle);
}
} while (::Process32Next(SnapShot, &Entry));
}
@ -1090,13 +1103,16 @@ void UBToolsBPLibrary::BringAppToForeground()
SetFocus(hWnd);
SetActiveWindow(hWnd);
AttachThreadInput(ForegroundThreadId, CurrentThreadId, false);
} else {
}
else {
UE_LOG(LogTemp, Warning, TEXT("BringAppToForeground: hWnd is null"));
}
} else {
}
else {
UE_LOG(LogTemp, Warning, TEXT("BringAppToForeground: NativeWindow invalid"));
}
} else {
}
else {
UE_LOG(LogTemp, Warning, TEXT("BringAppToForeground: MainWindow invalid"));
}
#endif

3
Unreal/Plugins/BTools/Source/BTools/Public/BToolsBPLibrary.h

@ -140,8 +140,9 @@ public:
UFUNCTION(BlueprintCallable, Category = "BTools")
static bool KillProcessByName(FString Name);
//Kill a process based on its path name. If UseAbolutePath is false all processes containing that FString dir will be killed.
UFUNCTION(BlueprintCallable, Category = "BTools")
static int KillProcessesByPath(FString Dir);
static int KillProcessesByPath(FString Dir, bool UseAbsolutePath = true);
UFUNCTION(BlueprintCallable, Category = "BTools")
static bool KillProcessesByID(int ProcessID);

4
Unreal/SPIE_Avatar.uproject

@ -65,6 +65,10 @@
{
"Name": "RigLogic",
"Enabled": true
},
{
"Name": "RawInput",
"Enabled": true
}
],
"TargetPlatforms": [

2
Unreal/SyncAvatarCore.bat

@ -49,7 +49,7 @@ if not exist "%SRC_ROOT%\" (
rem List of folders to mirror
set "FOLDERS=Plugins\AvatarCore_AI Plugins\AvatarCore_Manager Plugins\AvatarCore_MetaHuman Plugins\AvatarCore_STT Plugins\AvatarCore_TTS Plugins\BLogger Plugins\BSettings Plugins\BTools Plugins\RuntimeMetaHumanLipSync_5.6 Content\Project"
set "FOLDERS=Plugins\AvatarCore_AI Plugins\AvatarCore_Shared Plugins\AvatarCore_Manager Plugins\AvatarCore_MetaHuman Plugins\AvatarCore_STT Plugins\AvatarCore_TTS Plugins\BLogger Plugins\BSettings Plugins\BTools Plugins\RuntimeMetaHumanLipSync_5.6 Content\Project"
set "ANY_FAIL=0"

109
Unreal/SyncBackToAvatarCore.bat

@ -0,0 +1,109 @@
@echo off
setlocal EnableExtensions EnableDelayedExpansion
rem ============================================================
rem Mirror specific folders from A (AvatarBase) to B (this repo)
rem - Works relative to this .bat location
rem - Wipes extras in destination (/MIR)
rem - Logs removed extras (shown as *EXTRA File / *EXTRA Dir)
rem - Pauses before exit so console stays open
rem ============================================================
rem Destination root is the folder where this script lives
set "SRC_ROOT=%~dp0"
if "%SRC_ROOT:~-1%"=="\" set "SRC_ROOT=%SRC_ROOT:~0,-1%"
rem Source root relative to this script
set "DST_ROOT=%SRC_ROOT%\..\..\AvatarBase\Unreal"
rem Canonicalize paths
for %%I in ("%SRC_ROOT%") do set "SRC_ROOT=%%~fI"
for %%I in ("%DST_ROOT%") do set "DST_ROOT=%%~fI"
echo ============================================================
echo Robocopy Sync Script
echo ============================================================
echo Source: %SRC_ROOT%
echo Destination: %DST_ROOT%
echo ============================================================
echo.
rem Guard: only intended for subprojects
if /I "%SRC_ROOT%"=="%DST_ROOT%" (
echo ERROR: Source and destination are identical:
echo %SRC_ROOT%
echo This sync script is intended to be run from subprojects only.
echo.
pause
exit /b 1
)
if not exist "%SRC_ROOT%\" (
echo ERROR: Source root does not exist:
echo %SRC_ROOT%
echo.
pause
exit /b 2
)
echo ============================================================
echo Push back data from a project to avatar base? Are you sure? y/n
echo ============================================================
:PROMPT
SET /P AREYOUSURE=Are you sure (Y/[N])?
IF /I "%AREYOUSURE%" NEQ "Y" GOTO END
rem List of folders to mirror
set "FOLDERS=Plugins\AvatarCore_AI Plugins\AvatarCore_Shared Plugins\AvatarCore_Manager Plugins\AvatarCore_MetaHuman Plugins\AvatarCore_STT Plugins\AvatarCore_TTS Plugins\BLogger Plugins\BSettings Plugins\BTools Plugins\RuntimeMetaHumanLipSync_5.6 Content\Project"
set "ANY_FAIL=0"
for %%F in (%FOLDERS%) do (
set "SRC=%SRC_ROOT%\%%F"
set "DST=%DST_ROOT%\%%F"
echo ------------------------------------------------------------
echo Mirroring: %%F
echo FROM: !SRC!
echo TO: !DST!
echo ------------------------------------------------------------
if not exist "!SRC!\" (
echo WARNING: Source folder missing, skipping: !SRC!
) else (
if not exist "!DST!\" (
mkdir "!DST!" 2>nul
)
robocopy "!SRC!" "!DST!" /MIR /DCOPY:DAT /COPY:DAT /R:2 /W:1 /FFT /Z /NP /FP /TS /TEE
set "RC=!ERRORLEVEL!"
echo Robocopy exit code: !RC!
if !RC! GEQ 8 (
echo ERROR: Robocopy reported failure for %%F ^(exit code !RC!^)
set "ANY_FAIL=1"
)
)
echo.
)
if "!ANY_FAIL!"=="1" (
echo ============================================================
echo Sync failed!
echo ============================================================
pause
exit /b 8
)
echo ============================================================
echo Sync finished.
echo ============================================================
:END
endlocal
pause
exit /b 0
Loading…
Cancel
Save