Browse Source

Matched with Base

master
Tillman Staffen 1 week ago
parent
commit
06dff0030b
  1. 1
      Unreal/Config/DefaultInput.ini
  2. BIN
      Unreal/Content/Project/BP/BP_Project_Manager.uasset
  3. BIN
      Unreal/Content/Project/BP/EnumsAndStructs/S_DEMO_Settings.uasset
  4. BIN
      Unreal/Content/Project/Widgets/ChildWidgets/W_ToolcallEntry.uasset
  5. BIN
      Unreal/Content/Project/Widgets/Debug/W_DebugStateWidget.uasset
  6. BIN
      Unreal/Content/Project/Widgets/W_DialogueBox.uasset
  7. BIN
      Unreal/Content/Project/Widgets/W_Toolcall.uasset
  8. BIN
      Unreal/Content/Project/Widgets/W_Toolcall_Simple.uasset
  9. BIN
      Unreal/Content/SPIE/BP/BP_SPIE_Manager_Child.uasset
  10. BIN
      Unreal/Content/SPIE/BP/S_SPIE_ConfigSettings.uasset
  11. 132
      Unreal/Content/Schema/Spie_Config.schema.json
  12. 6
      Unreal/Plugins/AvatarCore_AI/AvatarCore_AI.uplugin
  13. 8
      Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/AvatarCore_AI.Build.cs
  14. 94
      Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/AIBaseManager.cpp
  15. 42
      Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/MCP/FastMCP/FastMCPManager.cpp
  16. 28
      Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/MCP/MCPUnrealCommand.cpp
  17. 4
      Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/RealtimeAPI/AvatarCoreAIRealtime.cpp
  18. 178
      Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/Tools/OpenAiModerator.cpp
  19. 7
      Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/Tools/OpenAiResponder.cpp
  20. 7
      Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/Tools/OpenRouterResponder.cpp
  21. 4
      Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/AIBaseConfig.h
  22. 43
      Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/AIBaseManager.h
  23. 32
      Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/MCP/FastMCP/FastMCPConfig.h
  24. 4
      Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/MCP/FastMCP/FastMCPManager.h
  25. 6
      Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/MCP/MCPBaseManager.h
  26. 7
      Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/MCP/MCPUnrealCommand.h
  27. 159
      Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/Tools/OpenAiModerator.h
  28. 6
      Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/Tools/OpenAiResponder.h
  29. 6
      Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/Tools/OpenRouterResponder.h
  30. 4
      Unreal/Plugins/AvatarCore_Manager/AvatarCore_Manager.uplugin
  31. BIN
      Unreal/Plugins/AvatarCore_Manager/Content/AvatarCoreManager.uasset
  32. BIN
      Unreal/Plugins/AvatarCore_Manager/Content/StateManagement/States/BP_BaseState.uasset
  33. BIN
      Unreal/Plugins/AvatarCore_Manager/Content/UnrealCommands/AICommand_CurrentLocation.uasset
  34. BIN
      Unreal/Plugins/AvatarCore_Manager/Content/UnrealCommands/DebutWait_Command.uasset
  35. BIN
      Unreal/Plugins/AvatarCore_Manager/Content/Widgets/Debug/Pages/ChildWidget/W_AutoTranslator.uasset
  36. BIN
      Unreal/Plugins/AvatarCore_Manager/Content/Widgets/Debug/Pages/W_DebugAvatarCoreSTT.uasset
  37. 1
      Unreal/Plugins/AvatarCore_Manager/Source/AvatarCore_Manager/AvatarCore_Manager.Build.cs
  38. 141
      Unreal/Plugins/AvatarCore_Manager/Source/AvatarCore_Manager/Private/FL_AvatarCoreManager.cpp
  39. 47
      Unreal/Plugins/AvatarCore_Manager/Source/AvatarCore_Manager/Public/AvatarCore_ManagerEnums.h
  40. 6
      Unreal/Plugins/AvatarCore_Manager/Source/AvatarCore_Manager/Public/FL_AvatarCoreManager.h
  41. 4
      Unreal/Plugins/AvatarCore_STT/AvatarCore_STT.uplugin
  42. 1
      Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/AvatarCore_STT.Build.cs
  43. 94
      Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Processor/Azure/STTProcessorAzure.cpp
  44. 20
      Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Processor/Parakeet/STTParakeetProcessorBase.cpp
  45. 16
      Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Processor/Whisper/STTProcessorWhisper.cpp
  46. 2
      Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/STTManagerBase.cpp
  47. 16
      Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Processor/Azure/STTAzureProcessorConfig.h
  48. 4
      Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Processor/Azure/STTProcessorAzure.h
  49. 30
      Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Processor/Parakeet/STTParakeetProcessorConfig.h
  50. 7
      Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Processor/STTBaseProcessorConfig.h
  51. 25
      Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Processor/Whisper/STTWhisperProcessorConfig.h
  52. 2
      Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/STTManagerBase.h
  53. 51
      Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/STTStructs.h
  54. 24
      Unreal/Plugins/AvatarCore_Shared/AvatarCore_Shared.uplugin
  55. BIN
      Unreal/Plugins/AvatarCore_Shared/Resources/Icon128.png
  56. 53
      Unreal/Plugins/AvatarCore_Shared/Source/AvatarCore_Shared/AvatarCore_Shared.Build.cs
  57. 20
      Unreal/Plugins/AvatarCore_Shared/Source/AvatarCore_Shared/Private/AvatarCore_Shared.cpp
  58. 27
      Unreal/Plugins/AvatarCore_Shared/Source/AvatarCore_Shared/Private/FL_AvatarCoreShared.cpp
  59. 62
      Unreal/Plugins/AvatarCore_Shared/Source/AvatarCore_Shared/Public/AvatarCoreSharedEnums.h
  60. 14
      Unreal/Plugins/AvatarCore_Shared/Source/AvatarCore_Shared/Public/AvatarCore_Shared.h
  61. 27
      Unreal/Plugins/AvatarCore_Shared/Source/AvatarCore_Shared/Public/FL_AvatarCoreShared.h
  62. 4
      Unreal/Plugins/AvatarCore_TTS/AvatarCore_TTS.uplugin
  63. 9
      Unreal/Plugins/AvatarCore_TTS/Source/.claude/settings.local.json
  64. 1
      Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/AvatarCore_TTS.Build.cs
  65. 68
      Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Private/Cartesia/CartesiaTTSManager.cpp
  66. 90
      Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Private/RealtimeAPI/RealtimeAPI_TTSManager.cpp
  67. 333
      Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Private/TTSManagerBase.cpp
  68. 35
      Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Private/TTSWebRTCChannel.cpp
  69. 11
      Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Public/RealtimeAPI/RealtimeAPI_TTSManager.h
  70. 73
      Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Public/TTSBaseConfig.h
  71. 36
      Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Public/TTSManagerBase.h
  72. 3
      Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Public/TTSWebRTCChannel.h
  73. 2
      Unreal/Plugins/BLogger/Source/BLogger/Public/SLogWidgetState.h
  74. BIN
      Unreal/Plugins/BTools/Content/Components/CheckProcessComponent.uasset
  75. 196
      Unreal/Plugins/BTools/Source/BTools/Private/BToolsBPLibrary.cpp
  76. 3
      Unreal/Plugins/BTools/Source/BTools/Public/BToolsBPLibrary.h
  77. 4
      Unreal/SPIE_Avatar.uproject
  78. 2
      Unreal/SyncAvatarCore.bat
  79. 109
      Unreal/SyncBackToAvatarCore.bat

1
Unreal/Config/DefaultInput.ini

@ -110,6 +110,7 @@ DoubleClickTime=0.200000
+ActionMappings=(ActionName="State7",bShift=False,bCtrl=False,bAlt=False,bCmd=False,Key=Seven) +ActionMappings=(ActionName="State7",bShift=False,bCtrl=False,bAlt=False,bCmd=False,Key=Seven)
+ActionMappings=(ActionName="State8",bShift=False,bCtrl=False,bAlt=False,bCmd=False,Key=Eight) +ActionMappings=(ActionName="State8",bShift=False,bCtrl=False,bAlt=False,bCmd=False,Key=Eight)
+ActionMappings=(ActionName="State9",bShift=False,bCtrl=False,bAlt=False,bCmd=False,Key=Nine) +ActionMappings=(ActionName="State9",bShift=False,bCtrl=False,bAlt=False,bCmd=False,Key=Nine)
+ActionMappings=(ActionName="Button",bShift=False,bCtrl=False,bAlt=False,bCmd=False,Key=GenericUSBController_Button12)
DefaultPlayerInputClass=/Script/EnhancedInput.EnhancedPlayerInput DefaultPlayerInputClass=/Script/EnhancedInput.EnhancedPlayerInput
DefaultInputComponentClass=/Script/EnhancedInput.EnhancedInputComponent DefaultInputComponentClass=/Script/EnhancedInput.EnhancedInputComponent
DefaultTouchInterface=/Engine/MobileResources/HUD/DefaultVirtualJoysticks.DefaultVirtualJoysticks DefaultTouchInterface=/Engine/MobileResources/HUD/DefaultVirtualJoysticks.DefaultVirtualJoysticks

BIN
Unreal/Content/Project/BP/BP_Project_Manager.uasset (Stored with Git LFS)

Binary file not shown.

BIN
Unreal/Content/Project/BP/EnumsAndStructs/S_DEMO_Settings.uasset (Stored with Git LFS)

Binary file not shown.

BIN
Unreal/Content/Project/Widgets/ChildWidgets/W_ToolcallEntry.uasset (Stored with Git LFS)

Binary file not shown.

BIN
Unreal/Content/Project/Widgets/Debug/W_DebugStateWidget.uasset (Stored with Git LFS)

Binary file not shown.

BIN
Unreal/Content/Project/Widgets/W_DialogueBox.uasset (Stored with Git LFS)

Binary file not shown.

BIN
Unreal/Content/Project/Widgets/W_Toolcall.uasset (Stored with Git LFS)

Binary file not shown.

BIN
Unreal/Content/Project/Widgets/W_Toolcall_Simple.uasset (Stored with Git LFS)

Binary file not shown.

BIN
Unreal/Content/SPIE/BP/BP_SPIE_Manager_Child.uasset (Stored with Git LFS)

Binary file not shown.

BIN
Unreal/Content/SPIE/BP/S_SPIE_ConfigSettings.uasset (Stored with Git LFS)

Binary file not shown.

132
Unreal/Content/Schema/Spie_Config.schema.json

@ -291,21 +291,123 @@
} }
}, },
{ {
"AzureSpeechService_API": "STTAzureSettings":
{ {
"type": "string", "type": "struct",
"tooltip": "Encrypted Azure API Key", "tooltip": "Settigns to configure the Mircrosoft Azure STT module",
"default": "0Gc0wvOF2tTCr4APvZkDaieKCGeBR9c5EbC5utXgVWZUqu4IAEf6NQ721iHNu74SwfDoYSBGJHLmCbcXVDP+F4HKnwsfHr7WYi9Gv+CjJ7/UrOygkqlrP05hbHBrJiPLWDv4Gw==", "fields":
"category": "STT Settings" {
"AzureAPIKey":
{
"type": "string"
},
"AzureRegion":
{
"type": "string"
}
},
"default":
{
"AzureAPIKey": "0Gc0wvOF2tTCr4APvZkDaieKCGeBR9c5EbC5utXgVWZUqu4IAEf6NQ721iHNu74SwfDoYSBGJHLmCbcXVDP+F4HKnwsfHr7WYi9Gv+CjJ7/UrOygkqlrP05hbHBrJiPLWDv4Gw==",
"AzureRegion": "germanywestcentral"
},
"category": "STT"
} }
}, },
{ {
"AzureSpeechService_Region": "STTWhisperSettings":
{ {
"type": "string", "type": "struct",
"tooltip": "Azure Region", "tooltip": "Settigns to configure the OpenAI Whisper STT module",
"default": "germanywestcentral", "fields":
"category": "STT Settings" {
"OpenAI_API_Key":
{
"type": "string"
},
"WhisperURL":
{
"type": "string"
},
"Model":
{
"type": "enum",
"enum": [
"Whisper-1",
"4o Transcribe Mini",
"4o Transcribe"
],
"enumTypeName": "EOpenAITranscriptionModel"
},
"MinDuration":
{
"type": "float"
}
},
"default":
{
"OpenAI_API_Key": "UjzfgavJ45lCu+oB2vVAsKNbPT+k3XCv7t69Og6j0LmwxhD3OK5WDBxUvgKnuDrz3xuNHg==",
"WhisperURL": "api.openai.com/v1/audio/transcriptions",
"Model": "4o Transcribe",
"MinDuration": 0.75
},
"category": "STT"
}
},
{
"STTParakeetSettings":
{
"type": "struct",
"tooltip": "Settigns to configure the nVidia parakeet STT module",
"fields":
{
"PythonPath":
{
"type": "string"
},
"PretrainedModel":
{
"type": "string"
},
"Host":
{
"type": "string"
},
"Port":
{
"type": "integer"
},
"KeepAliveRule":
{
"type": "enum",
"enum": [
"Only keep Module open in Editor Mode",
"Never keep Module alive.",
"Keep Module alive in Editor and Shipping mode."
],
"enumTypeName": "EKeepAliveRule"
},
"Device":
{
"type": "string"
},
"UpdateIntervalSec":
{
"type": "float",
"tooltip": "How often (seconds) the Python server produces intermediate transcription updates"
}
},
"default":
{
"PythonPath": "python",
"PretrainedModel": "nvidia/parakeet-tdt-0.6b-v3",
"Host": "127.0.0.1",
"Port": 40200,
"KeepAliveRule": "Only keep Module open in Editor Mode",
"Device": "cuda:0",
"UpdateIntervalSec": 0.5
},
"category": "STT"
} }
}, },
{ {
@ -562,7 +664,7 @@
"Marathi", "Marathi",
"Punjabi" "Punjabi"
], ],
"itemsEnumTypeName": "ESTTLanguage" "itemsEnumTypeName": "ELanguage"
}, },
"STTReplacements": "STTReplacements":
{ {
@ -785,11 +887,6 @@
"type": "struct", "type": "struct",
"fields": "fields":
{ {
"bUseModeration":
{
"type": "boolean",
"tooltip": "Check user transcription for inappropriate behaviour first (adds a delay!)"
},
"bUseMCPServer": "bUseMCPServer":
{ {
"type": "boolean", "type": "boolean",
@ -826,7 +923,6 @@
}, },
"default": "default":
{ {
"bUseModeration": false,
"bUseMCPServer": false, "bUseMCPServer": false,
"AIModelAudioOutput": true, "AIModelAudioOutput": true,
"MaxTokens": 1500, "MaxTokens": 1500,
@ -991,7 +1087,7 @@
"Marathi", "Marathi",
"Punjabi" "Punjabi"
], ],
"enumTypeName": "ETTSLanguage" "enumTypeName": "ELanguage"
} }
}, },
"default": "default":

6
Unreal/Plugins/AvatarCore_AI/AvatarCore_AI.uplugin

@ -5,7 +5,7 @@
"FriendlyName": "AvatarCore AI", "FriendlyName": "AvatarCore AI",
"Description": "A wrapper for OpenAI Assistents", "Description": "A wrapper for OpenAI Assistents",
"Category": "Other", "Category": "Other",
"CreatedBy": "b.RexGmbh", "CreatedBy": "b.ReX Gmbh",
"CreatedByURL": "", "CreatedByURL": "",
"DocsURL": "", "DocsURL": "",
"MarketplaceURL": "", "MarketplaceURL": "",
@ -29,6 +29,10 @@
{ {
"Name": "BTools", "Name": "BTools",
"Enabled": true "Enabled": true
},
{
"Name": "AvatarCore_Shared",
"Enabled": true
} }
] ]
} }

8
Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/AvatarCore_AI.Build.cs

@ -9,6 +9,11 @@ public class AvatarCore_AI : ModuleRules
{ {
PCHUsage = ModuleRules.PCHUsageMode.UseExplicitOrSharedPCHs; PCHUsage = ModuleRules.PCHUsageMode.UseExplicitOrSharedPCHs;
if (Target.bBuildEditor)
{
PublicDependencyModuleNames.Add("UnrealEd");
}
PublicIncludePaths.AddRange( PublicIncludePaths.AddRange(
new string[] { new string[] {
// ... add public include paths required here ... // ... add public include paths required here ...
@ -51,7 +56,8 @@ public class AvatarCore_AI : ModuleRules
"SSL", "SSL",
"Json", "Json",
"Projects", // Required for IPluginManager "Projects", // Required for IPluginManager
"BTools" "BTools",
"AvatarCore_Shared",
// ... add private dependencies that you statically link with here ... // ... add private dependencies that you statically link with here ...
} }
); );

94
Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/AIBaseManager.cpp

@ -1,6 +1,7 @@
// Fill out your copyright notice in the Description page of Project Settings. // Fill out your copyright notice in the Description page of Project Settings.
#include "AIBaseManager.h" #include "AIBaseManager.h"
#include "FL_AvatarCoreShared.h"
#include "Async/Async.h" #include "Async/Async.h"
void UAIBaseManager::InitAIManager(UAIBaseConfig* AIConfig, bool DebugMode, AActor* InWorldReferenceActor) void UAIBaseManager::InitAIManager(UAIBaseConfig* AIConfig, bool DebugMode, AActor* InWorldReferenceActor)
@ -77,6 +78,11 @@ void UAIBaseManager::InitAIManagerChild(UAIBaseConfig* AIConfig, AActor* InWorld
SetNewState(EAvatarCoreAIState::Ready); SetNewState(EAvatarCoreAIState::Ready);
} }
void UAIBaseManager::SetAILanguage(ELanguage NewLanguage)
{
ForcedLanguage = NewLanguage;
}
void UAIBaseManager::DeinitAIManager() void UAIBaseManager::DeinitAIManager()
{ {
PreviousMessages.Empty(); PreviousMessages.Empty();
@ -166,7 +172,13 @@ void UAIBaseManager::SendResponse(FAIMessage Message, bool NotifyDelay)
BroadcastAILog(FString::Printf(TEXT("AI Manager sent question/response: %s"), *Message.Message)); BroadcastAILog(FString::Printf(TEXT("AI Manager sent question/response: %s"), *Message.Message));
if (NotifyDelay) if (NotifyDelay)
UAIBaseManager::StartDelayedAnswerTimer(); UAIBaseManager::StartDelayedAnswerTimer();
//Add a fixed language tag. But only to the current message not to the stack. Do we really need this? Or does the system prompt is already enough?
FAIMessage currentMessage = Message;
if (ForcedLanguage != ELanguage::NONE && currentMessage.Role == EAvatarCoreAIPromptRole::User)
currentMessage.Message = "[Answer in " + UFL_AvatarCoreShared::LanguageToString(ForcedLanguage) + "]" + currentMessage.Message;
SendResponseChild(Message, NotifyDelay); SendResponseChild(Message, NotifyDelay);
if (Message.Role == EAvatarCoreAIPromptRole::User) if (Message.Role == EAvatarCoreAIPromptRole::User)
AddMessageToArray(Message); AddMessageToArray(Message);
else if (Message.Role == EAvatarCoreAIPromptRole::Tool) else if (Message.Role == EAvatarCoreAIPromptRole::Tool)
@ -186,10 +198,17 @@ void UAIBaseManager::RepeatText(FString TextToRepeat, bool DoRephrase)
AnswerCache.Empty(); AnswerCache.Empty();
ResponseID++; ResponseID++;
FString Instruction; FString Instruction;
if (DoRephrase) if (DoRephrase) {
Instruction = "[REPHRASE] " + TextToRepeat; if(ForcedLanguage == ELanguage::NONE)
Instruction = "[REPHRASE] " + TextToRepeat;
else
Instruction = "[REPHRASE in " + UFL_AvatarCoreShared::LanguageToString(ForcedLanguage) + "] " + TextToRepeat;
}
else else
Instruction = "[REPEAT] " + TextToRepeat; if (ForcedLanguage == ELanguage::NONE)
Instruction = "[REPEAT] " + TextToRepeat;
else
Instruction = "[REPEAT in " + UFL_AvatarCoreShared::LanguageToString(ForcedLanguage) + "] " + TextToRepeat;
FAIMessage tmpPrompt; FAIMessage tmpPrompt;
tmpPrompt.Message = Instruction; tmpPrompt.Message = Instruction;
tmpPrompt.Role = EAvatarCoreAIPromptRole::System; tmpPrompt.Role = EAvatarCoreAIPromptRole::System;
@ -317,7 +336,6 @@ void UAIBaseManager::RunMCPCommand(FString CommandName, FString Payload, FString
BroadcastAILog(FString::Printf(TEXT("Running Command '%s' with payload %s"), *CommandName, *Payload), true); BroadcastAILog(FString::Printf(TEXT("Running Command '%s' with payload %s"), *CommandName, *Payload), true);
SetNewState(EAvatarCoreAIState::GettingInfo); SetNewState(EAvatarCoreAIState::GettingInfo);
functionCallRunning = true;
if (FoundClass) if (FoundClass)
{ {
@ -329,7 +347,7 @@ void UAIBaseManager::RunMCPCommand(FString CommandName, FString Payload, FString
UMCPUnrealCommand* Cmd = NewObject<UMCPUnrealCommand>(this, FoundClass); UMCPUnrealCommand* Cmd = NewObject<UMCPUnrealCommand>(this, FoundClass);
Cmd->Id = ToolCallId; Cmd->Id = ToolCallId;
Cmd->SetWorldContext(WorldReferenceActor.Get()); Cmd->SetWorldContext(WorldReferenceActor.Get());
ActiveCommands.Add(Cmd); ActiveCommands.Add(ToolCallId, Cmd);
Cmd->OnCommandConfirmed.AddDynamic(this, &UAIBaseManager::CommandConfirmed); Cmd->OnCommandConfirmed.AddDynamic(this, &UAIBaseManager::CommandConfirmed);
Cmd->OnCommandDone.AddDynamic(this, &UAIBaseManager::CommandFinished); Cmd->OnCommandDone.AddDynamic(this, &UAIBaseManager::CommandFinished);
@ -337,28 +355,38 @@ void UAIBaseManager::RunMCPCommand(FString CommandName, FString Payload, FString
Cmd->InitMCPCommand(World); Cmd->InitMCPCommand(World);
Cmd->Execute(World, Payload); Cmd->Execute(World, Payload);
return;
} }
if (MCPManager && MCPManager->HasCommand(CommandName)) if (MCPManager && MCPManager->HasCommand(CommandName))
{ {
if (!ToolCallId.IsEmpty()) ActiveMCPCallIds.Add(ToolCallId);
MCPToolCallIds.Add(CommandName, ToolCallId); MCPManager->ExecuteCommand(CommandName, Payload, ToolCallId);
MCPManager->ExecuteCommand(CommandName, Payload);
} }
OnAIToolcallStarted.Broadcast(CommandName, ToolCallId, Payload);
} }
void UAIBaseManager::ClearMCPCommand() void UAIBaseManager::ClearMCPCommand()
{ {
for (UMCPUnrealCommand* Command : ActiveCommands) for (auto& Pair : ActiveCommands)
{ {
if (Command) if (Pair.Value)
{ {
Command->OnCommandDone.Clear(); Pair.Value->OnCommandDone.Clear();
Command->OnCommandFailed.Clear(); Pair.Value->OnCommandFailed.Clear();
} }
} }
ActiveCommands.Empty(); ActiveCommands.Empty();
ActiveMCPCallIds.Empty();
}
bool UAIBaseManager::AnyToolcallRunning() const
{
return ActiveCommands.Num() > 0 || ActiveMCPCallIds.Num() > 0;
}
int UAIBaseManager::HowManyToolcallRunning() const
{
return ActiveCommands.Num() + ActiveMCPCallIds.Num();
} }
FString UAIBaseManager::GetRoleAsString(EAvatarCoreAIPromptRole Role) FString UAIBaseManager::GetRoleAsString(EAvatarCoreAIPromptRole Role)
@ -384,13 +412,12 @@ void UAIBaseManager::CommandConfirmed(const FAIMessage& Message)
void UAIBaseManager::CommandFinished(const FAIMessage& Message) void UAIBaseManager::CommandFinished(const FAIMessage& Message)
{ {
ActiveCommands.RemoveAll([&Message](UMCPUnrealCommand* Cmd) UMCPUnrealCommand* Cmd = ActiveCommands.FindRef(Message.Id);
{ FString ToolName = Cmd ? Cmd->GetCommandName() : TEXT("");
return Cmd && (Message.Id.IsEmpty() || Cmd->Id.Equals(Message.Id)); ActiveCommands.Remove(Message.Id);
});
OnAIToolcallFinished.Broadcast(ToolName, Message.Id, Message.Message);
SetNewState(EAvatarCoreAIState::Ready); SetNewState(EAvatarCoreAIState::Ready);
functionCallRunning = false;
if (bDebugMode) if (bDebugMode)
BroadcastAILog(FString::Printf(TEXT("Command ran successfully. Answer: %s"), *Message.Message), true); BroadcastAILog(FString::Printf(TEXT("Command ran successfully. Answer: %s"), *Message.Message), true);
else else
@ -412,29 +439,21 @@ void UAIBaseManager::CommandFinished(const FAIMessage& Message)
void UAIBaseManager::CommandFailed(const FAIMessage& Message) void UAIBaseManager::CommandFailed(const FAIMessage& Message)
{ {
ActiveCommands.RemoveAll([&Message](UMCPUnrealCommand* Cmd) UMCPUnrealCommand* Cmd = ActiveCommands.FindRef(Message.Id);
{ FString ToolName = Cmd ? Cmd->GetCommandName() : TEXT("");
return Cmd && (Message.Id.IsEmpty() || Cmd->Id.Equals(Message.Id)); ActiveCommands.Remove(Message.Id);
});
functionCallRunning = false; OnAIToolcallError.Broadcast(ToolName, Message.Id, Message.Message);
SetNewState(EAvatarCoreAIState::Ready); SetNewState(EAvatarCoreAIState::Ready);
BroadcastAILog(FString::Printf(TEXT("Command failed. Sending: %s"), *Message.Message), true); BroadcastAILog(FString::Printf(TEXT("Command failed. Sending: %s"), *Message.Message), true);
SendResponse(Message, false); SendResponse(Message, false);
} }
void UAIBaseManager::MCPCommandFinished(const FString& Command, const FString& Payload) void UAIBaseManager::MCPCommandFinished(const FString& Command, const FString& Payload, const FString& ToolCallId)
{ {
FString FoundId; ActiveMCPCallIds.Remove(ToolCallId);
FString* MCPId = MCPToolCallIds.Find(Command); OnAIToolcallFinished.Broadcast(Command, ToolCallId, Payload);
if (MCPId)
{
FoundId = *MCPId;
MCPToolCallIds.Remove(Command);
}
SetNewState(EAvatarCoreAIState::Ready); SetNewState(EAvatarCoreAIState::Ready);
functionCallRunning = false;
if (bDebugMode) if (bDebugMode)
BroadcastAILog(FString::Printf(TEXT("MCP Command '%s' ran successfully. Answer: %s"), *Command, *Payload), true); BroadcastAILog(FString::Printf(TEXT("MCP Command '%s' ran successfully. Answer: %s"), *Command, *Payload), true);
else else
@ -442,10 +461,10 @@ void UAIBaseManager::MCPCommandFinished(const FString& Command, const FString& P
FAIMessage FinishedCommandMessage; FAIMessage FinishedCommandMessage;
FinishedCommandMessage.Message = Payload; FinishedCommandMessage.Message = Payload;
if (!FoundId.IsEmpty()) if (!ToolCallId.IsEmpty())
{ {
FinishedCommandMessage.Role = EAvatarCoreAIPromptRole::Tool; FinishedCommandMessage.Role = EAvatarCoreAIPromptRole::Tool;
FinishedCommandMessage.Id = FoundId; FinishedCommandMessage.Id = ToolCallId;
} }
else else
{ {
@ -454,9 +473,10 @@ void UAIBaseManager::MCPCommandFinished(const FString& Command, const FString& P
SendResponse(FinishedCommandMessage, false); SendResponse(FinishedCommandMessage, false);
} }
void UAIBaseManager::MCPCommandFailed(const FString& Command, const FString& Payload) void UAIBaseManager::MCPCommandFailed(const FString& Command, const FString& Payload, const FString& ToolCallId)
{ {
functionCallRunning = false; ActiveMCPCallIds.Remove(ToolCallId);
OnAIToolcallError.Broadcast(Command, ToolCallId, Payload);
SetNewState(EAvatarCoreAIState::Ready); SetNewState(EAvatarCoreAIState::Ready);
BroadcastAILog(FString::Printf(TEXT("MCP Command '%s' failed. Sending: %s"), *Command, *Payload), true); BroadcastAILog(FString::Printf(TEXT("MCP Command '%s' failed. Sending: %s"), *Command, *Payload), true);
FAIMessage FailedCommandMessage; FAIMessage FailedCommandMessage;

42
Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/MCP/FastMCP/FastMCPManager.cpp

@ -46,14 +46,31 @@ void UFastMCPManager::InitMCPManager(UMCPBaseConfig* InMCPConfig, bool DebugMode
return; return;
} }
bool bKeepAlive = false;
#if WITH_EDITOR #if WITH_EDITOR
bSpawnServerWhenNeeded = true; if (FastMCPConfig->FastMCPSettings.KeepAliveRule == EKeepAliveRule::Never)
CheckServerHealth(); bKeepAlive = false;
else
bKeepAlive = true;
#else #else
StopFastMCPServer(); if (FastMCPConfig->FastMCPSettings.KeepAliveRule == EKeepAliveRule::Always)
StartFastMCPServer(); bKeepAlive = true;
else
bKeepAlive = false;
#endif #endif
if(bKeepAlive)
{
bSpawnServerWhenNeeded = true;
CheckServerHealth();
}
else
{
StopFastMCPServer();
StartFastMCPServer();
}
} }
void UFastMCPManager::DeinitMCPManager() void UFastMCPManager::DeinitMCPManager()
@ -83,7 +100,7 @@ void UFastMCPManager::StartFastMCPServer()
LogMessage(TEXT("Starting FastMCP Server...")); LogMessage(TEXT("Starting FastMCP Server..."));
// Build command line arguments // Build command line arguments
FString Arguments = FString::Printf(TEXT("\"%s\" \"%s\""), *FastMCPConfig->PythonPath, *FastMCPConfig->Arguments); FString Arguments = FString::Printf(TEXT("\"%s\" \"%s\""), *FastMCPConfig->FastMCPSettings.PythonPath, *FastMCPConfig->FastMCPSettings.Arguments);
uint32 ProcId = 0; uint32 ProcId = 0;
@ -212,7 +229,7 @@ void UFastMCPManager::CheckServerHealth()
{ {
HealthCheckAttempts++; HealthCheckAttempts++;
if (HealthCheckAttempts > MaxHealthCheckAttempts) if (HealthCheckAttempts > FastMCPConfig->FastMCPSettings.MaxHealthCheckTimeInSec)
{ {
LogError(TEXT("FastMCP Server failed to respond after maximum attempts"), EMCPManagerError::InitializationError); LogError(TEXT("FastMCP Server failed to respond after maximum attempts"), EMCPManagerError::InitializationError);
if (UWorld* World = GEngine->GetWorldFromContextObject(this, EGetWorldErrorMode::LogAndReturnNull)) if (UWorld* World = GEngine->GetWorldFromContextObject(this, EGetWorldErrorMode::LogAndReturnNull))
@ -409,7 +426,7 @@ bool UFastMCPManager::ParseToolsFromResponse(const FString& JsonResponse, TArray
return false; return false;
} }
void UFastMCPManager::ExecuteCommand(const FString& Command, const FString& Payload) void UFastMCPManager::ExecuteCommand(const FString& Command, const FString& Payload, const FString& ToolCallId)
{ {
if (!bServerRunning) if (!bServerRunning)
{ {
@ -420,6 +437,7 @@ void UFastMCPManager::ExecuteCommand(const FString& Command, const FString& Payl
// Store current command for response handling // Store current command for response handling
CurrentExecutingCommand = Command; CurrentExecutingCommand = Command;
CurrentExecutingPayload = Payload; CurrentExecutingPayload = Payload;
CurrentToolCallId = ToolCallId;
SetState(EMCPManagerState::Busy); SetState(EMCPManagerState::Busy);
LogMessage(FString::Printf(TEXT("Executing command: %s"), *Command)); LogMessage(FString::Printf(TEXT("Executing command: %s"), *Command));
@ -428,7 +446,7 @@ void UFastMCPManager::ExecuteCommand(const FString& Command, const FString& Payl
TSharedPtr<FJsonObject> CallMessage = MakeShareable(new FJsonObject); TSharedPtr<FJsonObject> CallMessage = MakeShareable(new FJsonObject);
CallMessage->SetStringField(TEXT("jsonrpc"), TEXT("2.0")); CallMessage->SetStringField(TEXT("jsonrpc"), TEXT("2.0"));
CallMessage->SetStringField(TEXT("method"), TEXT("tools/call")); CallMessage->SetStringField(TEXT("method"), TEXT("tools/call"));
CallMessage->SetNumberField(TEXT("id"), 3); CallMessage->SetStringField(TEXT("id"), ToolCallId);
TSharedPtr<FJsonObject> Params = MakeShareable(new FJsonObject); TSharedPtr<FJsonObject> Params = MakeShareable(new FJsonObject);
Params->SetStringField(TEXT("name"), Command); Params->SetStringField(TEXT("name"), Command);
@ -473,14 +491,14 @@ void UFastMCPManager::OnCommandExecuted(FHttpRequestPtr Request, FHttpResponsePt
if (!bWasSuccessful || !Response.IsValid()) if (!bWasSuccessful || !Response.IsValid())
{ {
LogError(FString::Printf(TEXT("Failed to execute command: %s"), *CurrentExecutingCommand), EMCPManagerError::ToolError); LogError(FString::Printf(TEXT("Failed to execute command: %s"), *CurrentExecutingCommand), EMCPManagerError::ToolError);
OnMCPCommandFailed.Broadcast(CurrentExecutingCommand, CurrentExecutingPayload); OnMCPCommandFailed.Broadcast(CurrentExecutingCommand, CurrentExecutingPayload, CurrentToolCallId);
return; return;
} }
if (Response->GetResponseCode() != 200) if (Response->GetResponseCode() != 200)
{ {
LogError(FString::Printf(TEXT("Command execution failed with code %d: %s"), Response->GetResponseCode(), *CurrentExecutingCommand), EMCPManagerError::ToolError); LogError(FString::Printf(TEXT("Command execution failed with code %d: %s"), Response->GetResponseCode(), *CurrentExecutingCommand), EMCPManagerError::ToolError);
OnMCPCommandFailed.Broadcast(CurrentExecutingCommand, CurrentExecutingPayload); OnMCPCommandFailed.Broadcast(CurrentExecutingCommand, CurrentExecutingPayload, CurrentToolCallId);
return; return;
} }
@ -499,7 +517,7 @@ void UFastMCPManager::OnCommandExecuted(FHttpRequestPtr Request, FHttpResponsePt
if ((*ResultObject)->HasField(TEXT("isError")) && (*ResultObject)->GetBoolField(TEXT("isError"))) if ((*ResultObject)->HasField(TEXT("isError")) && (*ResultObject)->GetBoolField(TEXT("isError")))
{ {
LogError(FString::Printf(TEXT("Command execution failed: %s - %s"), *CurrentExecutingCommand, *ResponseContent), EMCPManagerError::ToolError); LogError(FString::Printf(TEXT("Command execution failed: %s - %s"), *CurrentExecutingCommand, *ResponseContent), EMCPManagerError::ToolError);
OnMCPCommandFailed.Broadcast(CurrentExecutingCommand, ResponseContent); OnMCPCommandFailed.Broadcast(CurrentExecutingCommand, ResponseContent, CurrentToolCallId);
return; return;
} }
} }
@ -508,7 +526,7 @@ void UFastMCPManager::OnCommandExecuted(FHttpRequestPtr Request, FHttpResponsePt
// Otherwise, broadcast success event with response // Otherwise, broadcast success event with response
LogMessage(FString::Printf(TEXT("Command executed successfully: %s"), *CurrentExecutingCommand)); LogMessage(FString::Printf(TEXT("Command executed successfully: %s"), *CurrentExecutingCommand));
OnMCPCommandDone.Broadcast(CurrentExecutingCommand, ResponseContent); OnMCPCommandDone.Broadcast(CurrentExecutingCommand, ResponseContent, CurrentToolCallId);
} }
void UFastMCPManager::SendMCPMessage(const FString& JsonMessage, TFunction<void(FHttpRequestPtr, FHttpResponsePtr, bool)> OnComplete) void UFastMCPManager::SendMCPMessage(const FString& JsonMessage, TFunction<void(FHttpRequestPtr, FHttpResponsePtr, bool)> OnComplete)

28
Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/MCP/MCPUnrealCommand.cpp

@ -5,6 +5,7 @@
void UMCPUnrealCommand::Execute_Implementation(UWorld* WorldReference, const FString& Payload) void UMCPUnrealCommand::Execute_Implementation(UWorld* WorldReference, const FString& Payload)
{ {
SetWorldContext(WorldReference);
StartTimeout(); StartTimeout();
} }
@ -114,9 +115,26 @@ UObject* UMCPUnrealCommand::GetWorldContextObject() const
UWorld* UMCPUnrealCommand::GetWorld() const UWorld* UMCPUnrealCommand::GetWorld() const
{ {
if (RequiredWorldContext) #if WITH_EDITOR
return RequiredWorldContext->GetWorld(); // Optional editor-only fallback before the game starts. This keeps the warning flood out of the way.
if (const UObject* Outer = GetOuter()) if (GEditor && HasAnyFlags(RF_ClassDefaultObject))
return Outer->GetWorld(); {
return nullptr; const FWorldContext& EditorWorldContext = GEditor->GetEditorWorldContext(false);
return EditorWorldContext.World();
}
// If it is not a ClassDefaultObject and still world is null, throw an error.
if (!RequiredWorldContext && !WarningMessageShown) {
FMessageLog("PIE").Error(FText::Format(
NSLOCTEXT("ObjectWithContext", "MissingWorldContext",
"WorldContextObject is not set on object '{0}' of class '{1}'."),
FText::FromString(GetName()),
FText::FromString(GetNameSafe(GetClass()))
));
FMessageLog("PIE").Notify();
WarningMessageShown = true;
}
#endif
return RequiredWorldContext ? RequiredWorldContext->GetWorld() : nullptr;
} }

4
Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/RealtimeAPI/AvatarCoreAIRealtime.cpp

@ -545,12 +545,12 @@ void UAvatarCoreAIRealtime::OnWebSocketConnectionStringReceived(const FString& M
else if (RequestItem == EOpenAIRequestItem::audio_transcript || RequestItem == EOpenAIRequestItem::text) else if (RequestItem == EOpenAIRequestItem::audio_transcript || RequestItem == EOpenAIRequestItem::text)
ResponseTextDone = true; ResponseTextDone = true;
if (functionCallRunning) { if (AnyToolcallRunning()) {
return; return;
} }
//float CurrentRequestDuration = (FDateTime::Now() - CurrentRequestStartTime).GetTotalSeconds(); //float CurrentRequestDuration = (FDateTime::Now() - CurrentRequestStartTime).GetTotalSeconds();
if (CurrentRequestDuration < 0.25f && !functionCallRunning && !CurrentRequestID.IsEmpty()) if (CurrentRequestDuration < 0.25f && !AnyToolcallRunning() && !CurrentRequestID.IsEmpty())
{ {
CurrentRequestID.Empty(); CurrentRequestID.Empty();
if (CurrentRetries < MaxRetries) if (CurrentRetries < MaxRetries)

178
Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/Tools/OpenAiModerator.cpp

@ -1,178 +0,0 @@
// Fill out your copyright notice in the Description page of Project Settings.
#include "Tools/OpenAiModerator.h"
#include "HttpModule.h"
#include "Interfaces/IHttpResponse.h"
#include "Dom/JsonObject.h"
#include "Serialization/JsonReader.h"
#include "Serialization/JsonSerializer.h"
FString UOpenAiModerator::StoredApiKey;
FString UOpenAiModerator::StoredModel = TEXT("omni-moderation-latest");
TArray<TWeakObjectPtr<UOpenAiModerator>> UOpenAiModerator::ActiveRequests;
void UOpenAiModerator::InitModeration(const FString& ApiKey, const FString& Model)
{
StoredApiKey = ApiKey;
StoredModel = Model;
}
UOpenAiModerator* UOpenAiModerator::UseModeration(UObject* WorldContextObject, const FString& Text)
{
UOpenAiModerator* Action = NewObject<UOpenAiModerator>();
Action->InputText = Text;
Action->RegisterWithGameInstance(WorldContextObject);
return Action;
}
void UOpenAiModerator::Activate()
{
if (StoredApiKey.IsEmpty())
{
UE_LOG(LogTemp, Error, TEXT("OpenAiModerator: API key not set. Call InitModeration first."));
FModerationResult EmptyResult;
Passed.Broadcast(EmptyResult);
return;
}
RequestStartTime = FPlatformTime::Seconds();
ActiveRequests.Add(this);
TSharedRef<IHttpRequest, ESPMode::ThreadSafe> HttpRequest = FHttpModule::Get().CreateRequest();
HttpRequest->SetURL(TEXT("https://api.openai.com/v1/moderations"));
HttpRequest->SetVerb(TEXT("POST"));
HttpRequest->SetHeader(TEXT("Content-Type"), TEXT("application/json"));
HttpRequest->SetHeader(TEXT("Authorization"), FString::Printf(TEXT("Bearer %s"), *StoredApiKey));
const FString Body = FString::Printf(TEXT("{\"model\":\"%s\",\"input\":\"%s\"}"),
*StoredModel,
*InputText.ReplaceCharWithEscapedChar());
HttpRequest->SetContentAsString(Body);
HttpRequest->OnProcessRequestComplete().BindUObject(this, &UOpenAiModerator::HandleHttpResponse);
if (!HttpRequest->ProcessRequest())
{
UE_LOG(LogTemp, Error, TEXT("OpenAiModerator: Failed to start HTTP request."));
FModerationResult EmptyResult;
Passed.Broadcast(EmptyResult);
}
}
void UOpenAiModerator::ClearModeration()
{
TArray<TWeakObjectPtr<UOpenAiModerator>> RequestsCopy = ActiveRequests;
ActiveRequests.Empty();
for (const TWeakObjectPtr<UOpenAiModerator>& Weak : RequestsCopy)
{
if (UOpenAiModerator* Action = Weak.Get())
{
Action->SetReadyToDestroy();
}
}
}
void UOpenAiModerator::HandleHttpResponse(FHttpRequestPtr Request, FHttpResponsePtr Response, bool bWasSuccessful)
{
ActiveRequests.RemoveAll([this](const TWeakObjectPtr<UOpenAiModerator>& Weak) { return !Weak.IsValid() || Weak.Get() == this; });
const float ElapsedTime = static_cast<float>(FPlatformTime::Seconds() - RequestStartTime);
FModerationResult Result;
Result.ExecutionTime = ElapsedTime;
if (!bWasSuccessful || !Response.IsValid())
{
UE_LOG(LogTemp, Error, TEXT("OpenAiModerator: HTTP request failed."));
Passed.Broadcast(Result);
return;
}
const int32 Code = Response->GetResponseCode();
const FString Body = Response->GetContentAsString();
if (Code < 200 || Code >= 300)
{
UE_LOG(LogTemp, Error, TEXT("OpenAiModerator: HTTP %d — %s"), Code, *Body);
Passed.Broadcast(Result);
return;
}
TSharedPtr<FJsonObject> RootObject;
TSharedRef<TJsonReader<>> Reader = TJsonReaderFactory<>::Create(Body);
if (!FJsonSerializer::Deserialize(Reader, RootObject) || !RootObject.IsValid())
{
UE_LOG(LogTemp, Error, TEXT("OpenAiModerator: Failed to parse response JSON."));
Passed.Broadcast(Result);
return;
}
const TArray<TSharedPtr<FJsonValue>>* ResultsArray;
if (!RootObject->TryGetArrayField(TEXT("results"), ResultsArray) || ResultsArray->Num() == 0)
{
UE_LOG(LogTemp, Error, TEXT("OpenAiModerator: No results in response."));
Passed.Broadcast(Result);
return;
}
const TSharedPtr<FJsonObject>& FirstResult = (*ResultsArray)[0]->AsObject();
if (!FirstResult.IsValid())
{
UE_LOG(LogTemp, Error, TEXT("OpenAiModerator: Invalid result object."));
Passed.Broadcast(Result);
return;
}
Result.bFlagged = FirstResult->GetBoolField(TEXT("flagged"));
const TSharedPtr<FJsonObject>* CategoriesObject;
if (FirstResult->TryGetObjectField(TEXT("categories"), CategoriesObject))
{
FModerationCategories& C = Result.Categories;
C.bSexual = (*CategoriesObject)->GetBoolField(TEXT("sexual"));
C.bSexualMinors = (*CategoriesObject)->GetBoolField(TEXT("sexual/minors"));
C.bHarassment = (*CategoriesObject)->GetBoolField(TEXT("harassment"));
C.bHarassmentThreatening = (*CategoriesObject)->GetBoolField(TEXT("harassment/threatening"));
C.bHate = (*CategoriesObject)->GetBoolField(TEXT("hate"));
C.bHateThreatening = (*CategoriesObject)->GetBoolField(TEXT("hate/threatening"));
C.bIllicit = (*CategoriesObject)->GetBoolField(TEXT("illicit"));
C.bIllicitViolent = (*CategoriesObject)->GetBoolField(TEXT("illicit/violent"));
C.bSelfHarm = (*CategoriesObject)->GetBoolField(TEXT("self-harm"));
C.bSelfHarmIntent = (*CategoriesObject)->GetBoolField(TEXT("self-harm/intent"));
C.bSelfHarmInstructions = (*CategoriesObject)->GetBoolField(TEXT("self-harm/instructions"));
C.bViolence = (*CategoriesObject)->GetBoolField(TEXT("violence"));
C.bViolenceGraphic = (*CategoriesObject)->GetBoolField(TEXT("violence/graphic"));
}
const TSharedPtr<FJsonObject>* ScoresObject;
if (FirstResult->TryGetObjectField(TEXT("category_scores"), ScoresObject))
{
FModerationCategoryScores& S = Result.CategoryScores;
S.Sexual = (*ScoresObject)->GetNumberField(TEXT("sexual"));
S.SexualMinors = (*ScoresObject)->GetNumberField(TEXT("sexual/minors"));
S.Harassment = (*ScoresObject)->GetNumberField(TEXT("harassment"));
S.HarassmentThreatening = (*ScoresObject)->GetNumberField(TEXT("harassment/threatening"));
S.Hate = (*ScoresObject)->GetNumberField(TEXT("hate"));
S.HateThreatening = (*ScoresObject)->GetNumberField(TEXT("hate/threatening"));
S.Illicit = (*ScoresObject)->GetNumberField(TEXT("illicit"));
S.IllicitViolent = (*ScoresObject)->GetNumberField(TEXT("illicit/violent"));
S.SelfHarm = (*ScoresObject)->GetNumberField(TEXT("self-harm"));
S.SelfHarmIntent = (*ScoresObject)->GetNumberField(TEXT("self-harm/intent"));
S.SelfHarmInstructions = (*ScoresObject)->GetNumberField(TEXT("self-harm/instructions"));
S.Violence = (*ScoresObject)->GetNumberField(TEXT("violence"));
S.ViolenceGraphic = (*ScoresObject)->GetNumberField(TEXT("violence/graphic"));
}
if (Result.bFlagged)
{
Flagged.Broadcast(Result);
}
else
{
Passed.Broadcast(Result);
}
}

7
Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/Tools/OpenAiResponder.cpp

@ -12,12 +12,14 @@
FString UOpenAiResponder::StoredApiKey; FString UOpenAiResponder::StoredApiKey;
FString UOpenAiResponder::StoredModel = TEXT("gpt-4.1"); FString UOpenAiResponder::StoredModel = TEXT("gpt-4.1");
bool UOpenAiResponder::IsResponderReady = false;
TArray<TWeakObjectPtr<UOpenAiResponder>> UOpenAiResponder::ActiveRequests; TArray<TWeakObjectPtr<UOpenAiResponder>> UOpenAiResponder::ActiveRequests;
void UOpenAiResponder::InitResponder(const FString& ApiKey, const FString& Model) void UOpenAiResponder::InitResponder(const FString& ApiKey, const FString& Model)
{ {
StoredApiKey = ApiKey; StoredApiKey = ApiKey;
StoredModel = Model; StoredModel = Model;
IsResponderReady = true;
} }
UOpenAiResponder* UOpenAiResponder::CallOpenAiResponse( UOpenAiResponder* UOpenAiResponder::CallOpenAiResponse(
@ -44,6 +46,11 @@ UOpenAiResponder* UOpenAiResponder::CallOpenAiResponse(
return Action; return Action;
} }
bool UOpenAiResponder::IsOpenAIResponderReady()
{
return IsResponderReady;
}
void UOpenAiResponder::Activate() void UOpenAiResponder::Activate()
{ {
if (StoredApiKey.IsEmpty()) if (StoredApiKey.IsEmpty())

7
Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/Tools/OpenRouterResponder.cpp

@ -15,6 +15,7 @@ FString UOpenRouterResponder::StoredModel = TEXT("openai/gpt-4.1");
FString UOpenRouterResponder::StoredSiteUrl; FString UOpenRouterResponder::StoredSiteUrl;
FString UOpenRouterResponder::StoredSiteName; FString UOpenRouterResponder::StoredSiteName;
TArray<TWeakObjectPtr<UOpenRouterResponder>> UOpenRouterResponder::ActiveRequests; TArray<TWeakObjectPtr<UOpenRouterResponder>> UOpenRouterResponder::ActiveRequests;
bool UOpenRouterResponder::IsResponderReady = false;
void UOpenRouterResponder::InitResponder(const FString& ApiKey, const FString& Model, const FString& SiteUrl, const FString& SiteName) void UOpenRouterResponder::InitResponder(const FString& ApiKey, const FString& Model, const FString& SiteUrl, const FString& SiteName)
{ {
@ -22,6 +23,7 @@ void UOpenRouterResponder::InitResponder(const FString& ApiKey, const FString& M
StoredModel = Model; StoredModel = Model;
StoredSiteUrl = SiteUrl; StoredSiteUrl = SiteUrl;
StoredSiteName = SiteName; StoredSiteName = SiteName;
IsResponderReady = true;
} }
UOpenRouterResponder* UOpenRouterResponder::CallOpenRouterResponse( UOpenRouterResponder* UOpenRouterResponder::CallOpenRouterResponse(
@ -46,6 +48,11 @@ UOpenRouterResponder* UOpenRouterResponder::CallOpenRouterResponse(
return Action; return Action;
} }
bool UOpenRouterResponder::IsOpenRouterResponderReady()
{
return IsResponderReady;
}
void UOpenRouterResponder::Activate() void UOpenRouterResponder::Activate()
{ {
if (StoredApiKey.IsEmpty()) if (StoredApiKey.IsEmpty())

4
Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/AIBaseConfig.h

@ -29,10 +29,6 @@ struct FGlobalAISettings
{ {
GENERATED_BODY() GENERATED_BODY()
// Check user transcription for inappropriate behaviour first (adds a delay!)
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|MCP", meta = (ExposeOnSpawn = "true"))
bool bUseModeration = false;
// Boot up a FastMCP Server on Startup (On close keep open in Editor Mode, kill in shipping) // Boot up a FastMCP Server on Startup (On close keep open in Editor Mode, kill in shipping)
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|MCP", meta = (ExposeOnSpawn = "true")) UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|MCP", meta = (ExposeOnSpawn = "true"))
bool bUseMCPServer = false; bool bUseMCPServer = false;

43
Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/AIBaseManager.h

@ -6,6 +6,7 @@
#include "UObject/Object.h" #include "UObject/Object.h"
#include "AIBaseConfig.h" #include "AIBaseConfig.h"
#include "AvatarCoreAIEnumsAndStructs.h" #include "AvatarCoreAIEnumsAndStructs.h"
#include "AvatarCoreSharedEnums.h"
#include "MCP/MCPBaseManager.h" #include "MCP/MCPBaseManager.h"
#include "AIBaseManager.generated.h" #include "AIBaseManager.generated.h"
@ -19,6 +20,9 @@ DECLARE_DYNAMIC_MULTICAST_DELEGATE_TwoParams(FOnAIError, FString, ErrorMessage,
DECLARE_DYNAMIC_MULTICAST_DELEGATE_TwoParams(FMulticastDelegateRealtimeAPIAudioChunk, const TArray<uint8>, PCMData, bool, IsFinal); DECLARE_DYNAMIC_MULTICAST_DELEGATE_TwoParams(FMulticastDelegateRealtimeAPIAudioChunk, const TArray<uint8>, PCMData, bool, IsFinal);
DECLARE_DYNAMIC_MULTICAST_DELEGATE(FOnAIDelayedAnswer); DECLARE_DYNAMIC_MULTICAST_DELEGATE(FOnAIDelayedAnswer);
DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FOnAIActivationStateChanged, bool, IsActive); DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FOnAIActivationStateChanged, bool, IsActive);
DECLARE_DYNAMIC_MULTICAST_DELEGATE_ThreeParams(FOnAIToolcallStarted, FString, ToolName, FString, ToolCallId, FString, Payload);
DECLARE_DYNAMIC_MULTICAST_DELEGATE_ThreeParams(FOnAIToolcallFinished, FString, ToolName, FString, ToolCallId, FString, Result);
DECLARE_DYNAMIC_MULTICAST_DELEGATE_ThreeParams(FOnAIToolcallError, FString, ToolName, FString, ToolCallId, FString, ErrorMessage);
class UMCPManager; class UMCPManager;
@ -57,6 +61,18 @@ public:
UPROPERTY(BlueprintAssignable, Category = "AvatarCoreAI|Events") UPROPERTY(BlueprintAssignable, Category = "AvatarCoreAI|Events")
FOnAIActivationStateChanged OnAIActivationStateChanged; FOnAIActivationStateChanged OnAIActivationStateChanged;
// Fired when a tool/function call is dispatched (Payload is raw JSON input)
UPROPERTY(BlueprintAssignable, Category = "AvatarCoreAI|Events")
FOnAIToolcallStarted OnAIToolcallStarted;
// Fired when a tool/function call completes successfully (Result is raw JSON output)
UPROPERTY(BlueprintAssignable, Category = "AvatarCoreAI|Events")
FOnAIToolcallFinished OnAIToolcallFinished;
// Fired when a tool/function call fails
UPROPERTY(BlueprintAssignable, Category = "AvatarCoreAI|Events")
FOnAIToolcallError OnAIToolcallError;
/** /**
* Initializes the AI Manager with the given config and adds this UObject to the root set. * Initializes the AI Manager with the given config and adds this UObject to the root set.
@ -67,6 +83,9 @@ public:
UFUNCTION(BlueprintCallable, Category = "AvatarCoreAI") UFUNCTION(BlueprintCallable, Category = "AvatarCoreAI")
virtual void InitAIManagerChild(UAIBaseConfig* AIConfig, AActor* InWorldReferenceActor); virtual void InitAIManagerChild(UAIBaseConfig* AIConfig, AActor* InWorldReferenceActor);
UFUNCTION(BlueprintCallable, Category = "AvatarCoreAI")
void SetAILanguage(ELanguage NewLanguage);
/** Returns the actor used as world context for commands */ /** Returns the actor used as world context for commands */
UFUNCTION(BlueprintPure, Category = "AvatarCoreAI") UFUNCTION(BlueprintPure, Category = "AvatarCoreAI")
AActor* GetWorldReferenceActor() const { return WorldReferenceActor.Get(); } AActor* GetWorldReferenceActor() const { return WorldReferenceActor.Get(); }
@ -196,6 +215,14 @@ public:
UFUNCTION() UFUNCTION()
void OnRequestTimeout(); void OnRequestTimeout();
// Returns true if any Unreal command instance is active or a function call is in progress
UFUNCTION(BlueprintPure, Category = "AvatarCoreAI")
bool AnyToolcallRunning() const;
// Returns amount of toolcalls (Unreal Command and MCP) that are currenlty active
UFUNCTION(BlueprintPure, Category = "AvatarCoreAI")
int HowManyToolcallRunning() const;
// Add a command at runtime (handles AddToRoot) // Add a command at runtime (handles AddToRoot)
UFUNCTION(BlueprintCallable, Category = "AvatarCoreAI|MCP") UFUNCTION(BlueprintCallable, Category = "AvatarCoreAI|MCP")
UMCPBaseManager* GetMCPManager(); UMCPBaseManager* GetMCPManager();
@ -242,11 +269,11 @@ protected:
/** Bound to MCPManager::OnMCPCommandDone — constructs FAIMessage from raw strings. */ /** Bound to MCPManager::OnMCPCommandDone — constructs FAIMessage from raw strings. */
UFUNCTION() UFUNCTION()
void MCPCommandFinished(const FString& Command, const FString& Payload); void MCPCommandFinished(const FString& Command, const FString& Payload, const FString& ToolCallId);
/** Bound to MCPManager::OnMCPCommandFailed. */ /** Bound to MCPManager::OnMCPCommandFailed. */
UFUNCTION() UFUNCTION()
void MCPCommandFailed(const FString& Command, const FString& Payload); void MCPCommandFailed(const FString& Command, const FString& Payload, const FString& ToolCallId);
//Add System/User/Assistant Message to memory archive //Add System/User/Assistant Message to memory archive
void AddMessageToArray(FAIMessage NewMessage); void AddMessageToArray(FAIMessage NewMessage);
@ -285,11 +312,10 @@ protected:
TArray<TSubclassOf<UMCPUnrealCommand>> UnrealCommandClasses; TArray<TSubclassOf<UMCPUnrealCommand>> UnrealCommandClasses;
TArray<FMCPToolInfo> UnrealCommandsToolInfos; TArray<FMCPToolInfo> UnrealCommandsToolInfos;
// Maps MCP server command name → tool_call_id for propagation through MCPCommandFinished
TMap<FString, FString> MCPToolCallIds;
UPROPERTY() UPROPERTY()
TArray<UMCPUnrealCommand*> ActiveCommands; TMap<FString, UMCPUnrealCommand*> ActiveCommands; // ToolCallId → Unreal command
TSet<FString> ActiveMCPCallIds; // ToolCallIds of in-flight MCP server commands
/** MCP Manager for FastMCP server communication */ /** MCP Manager for FastMCP server communication */
UPROPERTY() UPROPERTY()
@ -310,9 +336,6 @@ protected:
//Cached Answer //Cached Answer
int ResponseID = 0; int ResponseID = 0;
//There is a function call in progress
bool functionCallRunning = false;
//Current State the AI Manager //Current State the AI Manager
EAvatarCoreAIState CurrentAIState = EAvatarCoreAIState::Disconnected; EAvatarCoreAIState CurrentAIState = EAvatarCoreAIState::Disconnected;
@ -329,4 +352,6 @@ private:
TQueue<FAIMessage, EQueueMode::Spsc> ResponseQueue; TQueue<FAIMessage, EQueueMode::Spsc> ResponseQueue;
ELanguage ForcedLanguage = ELanguage::NONE;
}; };

32
Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/MCP/FastMCP/FastMCPConfig.h

@ -4,8 +4,31 @@
#include "CoreMinimal.h" #include "CoreMinimal.h"
#include "MCP/MCPBaseConfig.h" #include "MCP/MCPBaseConfig.h"
#include "AvatarCoreSharedEnums.h"
#include "FastMCPConfig.generated.h" #include "FastMCPConfig.generated.h"
USTRUCT(BlueprintType)
struct FFastMCPSettings
{
GENERATED_BODY()
//Custom python environment - "python" will use the system default
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|MCP", meta = (ExposeOnSpawn = "true"))
FString PythonPath = "python";
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|MCP", meta = (ExposeOnSpawn = "true"))
EKeepAliveRule KeepAliveRule = EKeepAliveRule::EditorOnly;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|MCP", meta = (ExposeOnSpawn = "true"))
int MaxHealthCheckTimeInSec = 180;
//Additional arguments to pass
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|MCP", meta = (ExposeOnSpawn = "true"))
FString Arguments = "";
};
/** /**
* *
*/ */
@ -18,14 +41,9 @@ public:
//Direction to the Script that start FastMCP //Direction to the Script that start FastMCP
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|MCP", meta = (ExposeOnSpawn = "true")) UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|MCP", meta = (ExposeOnSpawn = "true"))
FString MCPExecutable = FPaths::ProjectContentDir() +"DB/FastMCP/FastMCPServer.bat"; FString MCPExecutable = FPaths::ProjectContentDir() + "DB/FastMCP/FastMCPServer.bat";
//Custom python environment - "python" will use the system default
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreTTS|CoquiTTS", meta = (ExposeOnSpawn = "true"))
FString PythonPath = "python";
//Additional arguments to pass
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|MCP", meta = (ExposeOnSpawn = "true")) UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|MCP", meta = (ExposeOnSpawn = "true"))
FString Arguments = ""; FFastMCPSettings FastMCPSettings;
}; };

4
Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/MCP/FastMCP/FastMCPManager.h

@ -24,7 +24,7 @@ public:
virtual void InitMCPManager(UMCPBaseConfig* InMCPConfig, bool DebugMode) override; virtual void InitMCPManager(UMCPBaseConfig* InMCPConfig, bool DebugMode) override;
virtual void DeinitMCPManager() override; virtual void DeinitMCPManager() override;
virtual void FetchAvailableTools() override; virtual void FetchAvailableTools() override;
virtual void ExecuteCommand(const FString& Command, const FString& Payload) override; virtual void ExecuteCommand(const FString& Command, const FString& Payload, const FString& ToolCallId) override;
public: public:
// Helper to crop to first JSON object in a string (for SSE or noisy responses) // Helper to crop to first JSON object in a string (for SSE or noisy responses)
@ -67,9 +67,9 @@ private:
// Health check timer // Health check timer
FTimerHandle HealthCheckTimer; FTimerHandle HealthCheckTimer;
int32 HealthCheckAttempts; int32 HealthCheckAttempts;
static constexpr int32 MaxHealthCheckAttempts = 90; // 90 seconds max wait
// Current command tracking // Current command tracking
FString CurrentExecutingCommand; FString CurrentExecutingCommand;
FString CurrentExecutingPayload; FString CurrentExecutingPayload;
FString CurrentToolCallId;
}; };

6
Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/MCP/MCPBaseManager.h

@ -32,8 +32,8 @@ DECLARE_DYNAMIC_MULTICAST_DELEGATE_TwoParams(FOnMCPManagerError, const FString&,
DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FOnMCPManagerStateChanged, EMCPManagerState, NewMCPManagerState); DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FOnMCPManagerStateChanged, EMCPManagerState, NewMCPManagerState);
DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FOnMCPLog, const FString&, LogMessage); DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FOnMCPLog, const FString&, LogMessage);
DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FOnMCPToolsUpdated, const TArray<FMCPToolInfo>&, AvailableTools); DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FOnMCPToolsUpdated, const TArray<FMCPToolInfo>&, AvailableTools);
DECLARE_DYNAMIC_MULTICAST_DELEGATE_TwoParams(FOnMCPCommandDone, const FString&, Command, const FString&, Payload); DECLARE_DYNAMIC_MULTICAST_DELEGATE_ThreeParams(FOnMCPCommandDone, const FString&, Command, const FString&, Payload, const FString&, ToolCallId);
DECLARE_DYNAMIC_MULTICAST_DELEGATE_TwoParams(FOnMCPCommandFailed, const FString&, Command, const FString&, Payload); DECLARE_DYNAMIC_MULTICAST_DELEGATE_ThreeParams(FOnMCPCommandFailed, const FString&, Command, const FString&, Payload, const FString&, ToolCallId);
UCLASS(Abstract, Blueprintable, BlueprintType) UCLASS(Abstract, Blueprintable, BlueprintType)
class AVATARCORE_AI_API UMCPBaseManager : public UObject class AVATARCORE_AI_API UMCPBaseManager : public UObject
@ -81,7 +81,7 @@ public:
bool HasCommand(const FString& Command); bool HasCommand(const FString& Command);
UFUNCTION(BlueprintCallable, Category = "AvatarCoreAI|MCP|Operations") UFUNCTION(BlueprintCallable, Category = "AvatarCoreAI|MCP|Operations")
virtual void ExecuteCommand(const FString& Command, const FString& Payload) {}; virtual void ExecuteCommand(const FString& Command, const FString& Payload, const FString& ToolCallId) {};
// Blueprint Functions - State Management // Blueprint Functions - State Management
UFUNCTION(BlueprintCallable, BlueprintPure, Category = "AvatarCoreAI|MCP|State") UFUNCTION(BlueprintCallable, BlueprintPure, Category = "AvatarCoreAI|MCP|State")

7
Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/MCP/MCPUnrealCommand.h

@ -37,6 +37,8 @@ public:
UFUNCTION(BlueprintCallable, Category = "Context") UFUNCTION(BlueprintCallable, Category = "Context")
UObject* GetWorldContextObject() const; UObject* GetWorldContextObject() const;
virtual UWorld* GetWorld() const override;
// Result event (success) // Result event (success)
UPROPERTY(BlueprintAssignable, Category = "Command") UPROPERTY(BlueprintAssignable, Category = "Command")
FOnAICommandDone OnCommandDone; FOnAICommandDone OnCommandDone;
@ -103,8 +105,9 @@ protected:
UFUNCTION(BlueprintCallable, Category = "Command") UFUNCTION(BlueprintCallable, Category = "Command")
AActor* GetActorOfClass(UWorld* World, TSubclassOf<AActor> ActorClass) const; AActor* GetActorOfClass(UWorld* World, TSubclassOf<AActor> ActorClass) const;
virtual UWorld* GetWorld() const override;
void StartTimeout(); void StartTimeout();
void OnTimeout(); void OnTimeout();
private:
mutable bool WarningMessageShown;
}; };

159
Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/Tools/OpenAiModerator.h

@ -1,159 +0,0 @@
// Fill out your copyright notice in the Description page of Project Settings.
#pragma once
#include "CoreMinimal.h"
#include "Kismet/BlueprintAsyncActionBase.h"
#include "Http.h"
#include "OpenAiModerator.generated.h"
USTRUCT(BlueprintType)
struct FModerationCategories
{
GENERATED_BODY()
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation")
bool bSexual = false;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation")
bool bSexualMinors = false;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation")
bool bHarassment = false;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation")
bool bHarassmentThreatening = false;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation")
bool bHate = false;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation")
bool bHateThreatening = false;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation")
bool bIllicit = false;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation")
bool bIllicitViolent = false;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation")
bool bSelfHarm = false;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation")
bool bSelfHarmIntent = false;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation")
bool bSelfHarmInstructions = false;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation")
bool bViolence = false;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation")
bool bViolenceGraphic = false;
};
USTRUCT(BlueprintType)
struct FModerationCategoryScores
{
GENERATED_BODY()
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation")
float Sexual = 0.f;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation")
float SexualMinors = 0.f;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation")
float Harassment = 0.f;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation")
float HarassmentThreatening = 0.f;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation")
float Hate = 0.f;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation")
float HateThreatening = 0.f;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation")
float Illicit = 0.f;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation")
float IllicitViolent = 0.f;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation")
float SelfHarm = 0.f;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation")
float SelfHarmIntent = 0.f;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation")
float SelfHarmInstructions = 0.f;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation")
float Violence = 0.f;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation")
float ViolenceGraphic = 0.f;
};
USTRUCT(BlueprintType)
struct FModerationResult
{
GENERATED_BODY()
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation")
bool bFlagged = false;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation")
FModerationCategories Categories;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation")
FModerationCategoryScores CategoryScores;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation")
float ExecutionTime = 0.f;
};
DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FOnModerationResult, const FModerationResult&, Result);
/**
* Async Blueprint node for OpenAI Moderation API.
* Call InitModeration once to set API key and model, then use UseModeration to classify text.
*/
UCLASS()
class AVATARCORE_AI_API UOpenAiModerator : public UBlueprintAsyncActionBase
{
GENERATED_BODY()
public:
/** Set API key and model. Call once before using UseModeration. */
UFUNCTION(BlueprintCallable, Category = "AvatarCoreAI|Moderation", meta = (DisplayName = "Init Moderation"))
static void InitModeration(const FString& ApiKey, const FString& Model = TEXT("omni-moderation-latest"));
/** Classify text using the OpenAI Moderation API. Returns via Flagged or Passed exec pins. */
UFUNCTION(BlueprintCallable, Category = "AvatarCoreAI|Moderation", meta = (BlueprintInternalUseOnly = "true", DisplayName = "Use Moderation", WorldContext = "WorldContextObject"))
static UOpenAiModerator* UseModeration(UObject* WorldContextObject, const FString& Text);
/** Cancel all active moderation requests. */
UFUNCTION(BlueprintCallable, Category = "AvatarCoreAI|Moderation", meta = (DisplayName = "Clear Moderation"))
static void ClearModeration();
UPROPERTY(BlueprintAssignable)
FOnModerationResult Flagged;
UPROPERTY(BlueprintAssignable)
FOnModerationResult Passed;
virtual void Activate() override;
private:
void HandleHttpResponse(FHttpRequestPtr Request, FHttpResponsePtr Response, bool bWasSuccessful);
FString InputText;
double RequestStartTime = 0.0;
static FString StoredApiKey;
static FString StoredModel;
static TArray<TWeakObjectPtr<UOpenAiModerator>> ActiveRequests;
};

6
Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/Tools/OpenAiResponder.h

@ -123,6 +123,10 @@ public:
bool bUseWebSearch = false, bool bUseWebSearch = false,
EResponderToolChoice ToolChoice = EResponderToolChoice::None); EResponderToolChoice ToolChoice = EResponderToolChoice::None);
/** Is responder ready and api key set? */
UFUNCTION(BlueprintPure, Category = "AvatarCoreAI|Responder", meta = (DisplayName = "Is OpenAI Responder ready?"))
static bool IsOpenAIResponderReady();
/** Cancel all active response requests. */ /** Cancel all active response requests. */
UFUNCTION(BlueprintCallable, Category = "AvatarCoreAI|Responder", meta = (DisplayName = "Clear All Responses")) UFUNCTION(BlueprintCallable, Category = "AvatarCoreAI|Responder", meta = (DisplayName = "Clear All Responses"))
static void ClearAllResponses(); static void ClearAllResponses();
@ -148,6 +152,8 @@ private:
EResponderToolChoice InputToolChoice = EResponderToolChoice::None; EResponderToolChoice InputToolChoice = EResponderToolChoice::None;
double RequestStartTime = 0.0; double RequestStartTime = 0.0;
static bool IsResponderReady;
static FString StoredApiKey; static FString StoredApiKey;
static FString StoredModel; static FString StoredModel;
static TArray<TWeakObjectPtr<UOpenAiResponder>> ActiveRequests; static TArray<TWeakObjectPtr<UOpenAiResponder>> ActiveRequests;

6
Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/Tools/OpenRouterResponder.h

@ -104,6 +104,10 @@ public:
ERouterReasoning Reasoning = ERouterReasoning::None, ERouterReasoning Reasoning = ERouterReasoning::None,
ERouterToolChoice ToolChoice = ERouterToolChoice::Auto); ERouterToolChoice ToolChoice = ERouterToolChoice::Auto);
/** Is responder ready and api key set? */
UFUNCTION(BlueprintPure, Category = "AvatarCoreAI|Router", meta = (DisplayName = "Is OpenRouter Responder ready?"))
static bool IsOpenRouterResponderReady();
/** Cancel all active OpenRouter requests. */ /** Cancel all active OpenRouter requests. */
UFUNCTION(BlueprintCallable, Category = "AvatarCoreAI|Router", meta = (DisplayName = "Clear All Router Responses")) UFUNCTION(BlueprintCallable, Category = "AvatarCoreAI|Router", meta = (DisplayName = "Clear All Router Responses"))
static void ClearAllResponses(); static void ClearAllResponses();
@ -128,6 +132,8 @@ private:
ERouterToolChoice InputToolChoice = ERouterToolChoice::Auto; ERouterToolChoice InputToolChoice = ERouterToolChoice::Auto;
double RequestStartTime = 0.0; double RequestStartTime = 0.0;
static bool IsResponderReady;
static FString StoredApiKey; static FString StoredApiKey;
static FString StoredModel; static FString StoredModel;
static FString StoredSiteUrl; static FString StoredSiteUrl;

4
Unreal/Plugins/AvatarCore_Manager/AvatarCore_Manager.uplugin

@ -37,6 +37,10 @@
{ {
"Name": "AvatarCore_TTS", "Name": "AvatarCore_TTS",
"Enabled": true "Enabled": true
},
{
"Name": "AvatarCore_Shared",
"Enabled": true
} }
] ]
} }

BIN
Unreal/Plugins/AvatarCore_Manager/Content/AvatarCoreManager.uasset (Stored with Git LFS)

Binary file not shown.

BIN
Unreal/Plugins/AvatarCore_Manager/Content/StateManagement/States/BP_BaseState.uasset (Stored with Git LFS)

Binary file not shown.

BIN
Unreal/Plugins/AvatarCore_Manager/Content/UnrealCommands/AICommand_CurrentLocation.uasset (Stored with Git LFS)

Binary file not shown.

BIN
Unreal/Plugins/AvatarCore_Manager/Content/UnrealCommands/DebutWait_Command.uasset (Stored with Git LFS)

Binary file not shown.

BIN
Unreal/Plugins/AvatarCore_Manager/Content/Widgets/Debug/Pages/ChildWidget/W_AutoTranslator.uasset (Stored with Git LFS)

Binary file not shown.

BIN
Unreal/Plugins/AvatarCore_Manager/Content/Widgets/Debug/Pages/W_DebugAvatarCoreSTT.uasset (Stored with Git LFS)

Binary file not shown.

1
Unreal/Plugins/AvatarCore_Manager/Source/AvatarCore_Manager/AvatarCore_Manager.Build.cs

@ -42,6 +42,7 @@ public class AvatarCore_Manager : ModuleRules
"AvatarCore_TTS", "AvatarCore_TTS",
"AvatarCore_STT", "AvatarCore_STT",
"AvatarCore_AI", "AvatarCore_AI",
"AvatarCore_Shared",
"DeveloperSettings" "DeveloperSettings"
// ... add private dependencies that you statically link with here ... // ... add private dependencies that you statically link with here ...
} }

141
Unreal/Plugins/AvatarCore_Manager/Source/AvatarCore_Manager/Private/FL_AvatarCoreManager.cpp

@ -17,144 +17,3 @@ void UFL_AvatarCoreManager::BindTTSToAIManager(UTTSManagerBase* TTSManager, UAIB
AIManager->OnAudioChunk.AddDynamic(TTSManager, &UTTSManagerBase::AddAudioChunk); AIManager->OnAudioChunk.AddDynamic(TTSManager, &UTTSManagerBase::AddAudioChunk);
} }
} }
ETTSLanguage UFL_AvatarCoreManager::ConvertLanguageToTTS(ELanguage Language)
{
switch (Language)
{
case ELanguage::ar:
return ETTSLanguage::ar;
case ELanguage::de:
return ETTSLanguage::de;
case ELanguage::en:
return ETTSLanguage::en;
case ELanguage::fr:
return ETTSLanguage::fr;
case ELanguage::el:
return ETTSLanguage::el;
case ELanguage::hi:
return ETTSLanguage::hi;
case ELanguage::it:
return ETTSLanguage::it;
case ELanguage::ja:
return ETTSLanguage::ja;
case ELanguage::ko:
return ETTSLanguage::ko;
case ELanguage::zh:
return ETTSLanguage::zh;
case ELanguage::nl:
return ETTSLanguage::nl;
case ELanguage::pl:
return ETTSLanguage::pl;
case ELanguage::pt:
return ETTSLanguage::pt;
case ELanguage::ro:
return ETTSLanguage::ro;
case ELanguage::ru:
return ETTSLanguage::ru;
case ELanguage::es:
return ETTSLanguage::es;
case ELanguage::cs:
return ETTSLanguage::cs;
case ELanguage::tr:
return ETTSLanguage::tr;
case ELanguage::uk:
return ETTSLanguage::uk;
case ELanguage::hu:
return ETTSLanguage::hu;
case ELanguage::NONE:
default:
return ETTSLanguage::NONE;
}
}
ESTTLanguage UFL_AvatarCoreManager::ConvertLanguageToSTT(ELanguage Language)
{
switch (Language)
{
case ELanguage::ar:
return ESTTLanguage::ar;
case ELanguage::de:
return ESTTLanguage::de;
case ELanguage::en:
return ESTTLanguage::en;
case ELanguage::fr:
return ESTTLanguage::fr;
case ELanguage::el:
return ESTTLanguage::el;
case ELanguage::hi:
return ESTTLanguage::hi;
case ELanguage::it:
return ESTTLanguage::it;
case ELanguage::ja:
return ESTTLanguage::ja;
case ELanguage::ko:
return ESTTLanguage::ko;
case ELanguage::zh:
return ESTTLanguage::zh;
case ELanguage::nl:
return ESTTLanguage::nl;
case ELanguage::pl:
return ESTTLanguage::pl;
case ELanguage::pt:
return ESTTLanguage::pt;
case ELanguage::ro:
return ESTTLanguage::ro;
case ELanguage::ru:
return ESTTLanguage::ru;
case ELanguage::es:
return ESTTLanguage::es;
case ELanguage::cs:
return ESTTLanguage::cs;
case ELanguage::tr:
return ESTTLanguage::tr;
case ELanguage::uk:
return ESTTLanguage::uk;
case ELanguage::hu:
return ESTTLanguage::hu;
case ELanguage::NONE:
default:
return ESTTLanguage::NONE;
}
}

47
Unreal/Plugins/AvatarCore_Manager/Source/AvatarCore_Manager/Public/AvatarCore_ManagerEnums.h

@ -32,51 +32,4 @@ enum class EAvatarState : uint8 {
Talking UMETA(DisplayName = "Avatar talking"), Talking UMETA(DisplayName = "Avatar talking"),
IdleActive UMETA(DisplayName = "Avatar idle active"), IdleActive UMETA(DisplayName = "Avatar idle active"),
IdlePassive UMETA(DisplayName = "Avatar idle passive") IdlePassive UMETA(DisplayName = "Avatar idle passive")
};
UENUM(BlueprintType)
enum class ELanguage : uint8 {
NONE UMETA(DisplayName = "Unset"),
en UMETA(DisplayName = "English"),
fr UMETA(DisplayName = "French"),
de UMETA(DisplayName = "German"),
es UMETA(DisplayName = "Spanish"),
pt UMETA(DisplayName = "Portuguese"),
zh UMETA(DisplayName = "Chinese"),
ja UMETA(DisplayName = "Japanese"),
hi UMETA(DisplayName = "Hindi"),
it UMETA(DisplayName = "Italian"),
ko UMETA(DisplayName = "Korean"),
nl UMETA(DisplayName = "Dutch"),
pl UMETA(DisplayName = "Polish"),
ru UMETA(DisplayName = "Russian"),
sv UMETA(DisplayName = "Swedish"),
tr UMETA(DisplayName = "Turkish"),
tl UMETA(DisplayName = "Filipino"),
bg UMETA(DisplayName = "Bulgarian"),
ro UMETA(DisplayName = "Romanian"),
ar UMETA(DisplayName = "Arabic"),
cs UMETA(DisplayName = "Czech"),
el UMETA(DisplayName = "Greek"),
fi UMETA(DisplayName = "Finnish"),
hr UMETA(DisplayName = "Croatian"),
ms UMETA(DisplayName = "Malay"),
sk UMETA(DisplayName = "Slovak"),
da UMETA(DisplayName = "Danish"),
ta UMETA(DisplayName = "Tamil"),
uk UMETA(DisplayName = "Ukrainian"),
hu UMETA(DisplayName = "Hungarian"),
no UMETA(DisplayName = "Norwegian"),
vi UMETA(DisplayName = "Vietnamese"),
bn UMETA(DisplayName = "Bengali"),
th UMETA(DisplayName = "Thai"),
he UMETA(DisplayName = "Hebrew"),
ka UMETA(DisplayName = "Georgian"),
id UMETA(DisplayName = "Indonesian"),
te UMETA(DisplayName = "Telugu"),
gu UMETA(DisplayName = "Gujarati"),
kn UMETA(DisplayName = "Kannada"),
ml UMETA(DisplayName = "Malayalam"),
mr UMETA(DisplayName = "Marathi"),
pa UMETA(DisplayName = "Punjabi"),
}; };

6
Unreal/Plugins/AvatarCore_Manager/Source/AvatarCore_Manager/Public/FL_AvatarCoreManager.h

@ -48,10 +48,4 @@ class AVATARCORE_MANAGER_API UFL_AvatarCoreManager : public UBlueprintFunctionLi
UFUNCTION(BlueprintCallable, Category = "AvatarCoreManager") UFUNCTION(BlueprintCallable, Category = "AvatarCoreManager")
static void BindTTSToAIManager(UTTSManagerBase* TTSManager, UAIBaseManager* AIManager); static void BindTTSToAIManager(UTTSManagerBase* TTSManager, UAIBaseManager* AIManager);
UFUNCTION(BlueprintCallable, Category = "AvatarCoreManager")
static ETTSLanguage ConvertLanguageToTTS(ELanguage Language);
UFUNCTION(BlueprintCallable, Category = "AvatarCoreManager")
static ESTTLanguage ConvertLanguageToSTT(ELanguage Language);
}; };

4
Unreal/Plugins/AvatarCore_STT/AvatarCore_STT.uplugin

@ -29,6 +29,10 @@
{ {
"Name": "AudioCapture", "Name": "AudioCapture",
"Enabled": true "Enabled": true
},
{
"Name": "AvatarCore_Shared",
"Enabled": true
} }
] ]
} }

1
Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/AvatarCore_STT.Build.cs

@ -101,6 +101,7 @@ public class AvatarCore_STT : ModuleRules
"Json", "Json",
"JsonUtilities", "JsonUtilities",
"WebRTC", "WebRTC",
"AvatarCore_Shared",
// ... add private dependencies that you statically link with here ... // ... add private dependencies that you statically link with here ...
} }
); );

94
Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Processor/Azure/STTProcessorAzure.cpp

@ -21,14 +21,14 @@ void USTTProcessorAzure::InitSTTProcessor(USTTManagerBase* BaseSTTManager, USTTB
return; return;
} }
if (AzureProcessorConfig->AzureAPIKey.IsEmpty()) { if (AzureProcessorConfig->AzureSettings.AzureAPIKey.IsEmpty()) {
STTManager->OnSTTError.Broadcast(TEXT("Azure API Key not set. Needs to be done before initializing modules.")); STTManager->OnSTTError.Broadcast(TEXT("Azure API Key not set. Needs to be done before initializing modules."));
return; return;
} }
// Convert FString to std::string // Convert FString to std::string
std::string SubscriptionKey = TCHAR_TO_UTF8(*AzureProcessorConfig->AzureAPIKey); std::string SubscriptionKey = TCHAR_TO_UTF8(*AzureProcessorConfig->AzureSettings.AzureAPIKey);
std::string Region = TCHAR_TO_UTF8(*AzureProcessorConfig->AzureRegion); std::string Region = TCHAR_TO_UTF8(*AzureProcessorConfig->AzureSettings.AzureRegion);
// Create the SpeechConfig object // Create the SpeechConfig object
config = SpeechSDK::SpeechConfig::FromSubscription(SubscriptionKey, Region); config = SpeechSDK::SpeechConfig::FromSubscription(SubscriptionKey, Region);
@ -89,7 +89,7 @@ void USTTProcessorAzure::OnChunkReceived(TArray<int16> PCMData, FAudioInformatio
StopRecognition(false); StopRecognition(false);
} }
void USTTProcessorAzure::ChangeAzureLanguage(TArray<ESTTLanguage> InLanguages) void USTTProcessorAzure::ChangeAzureLanguage(TArray<ELanguage> InLanguages)
{ {
AzureProcessorConfig->BaseSettings.STTLanguages = InLanguages; AzureProcessorConfig->BaseSettings.STTLanguages = InLanguages;
if (bDebugMode && IsValid(STTManager)) if (bDebugMode && IsValid(STTManager))
@ -232,52 +232,52 @@ void USTTProcessorAzure::OnAzureError(FString Error)
} }
} }
FString USTTProcessorAzure::AzureEnumToString(ESTTLanguage Language) FString USTTProcessorAzure::AzureEnumToString(ELanguage Language)
{ {
switch (Language) switch (Language)
{ {
case ESTTLanguage::en: return "en-US"; case ELanguage::en: return "en-US";
case ESTTLanguage::de: return "de-DE"; case ELanguage::de: return "de-DE";
case ESTTLanguage::fr: return "fr-FR"; case ELanguage::fr: return "fr-FR";
case ESTTLanguage::es: return "es-ES"; case ELanguage::es: return "es-ES";
case ESTTLanguage::pt: return "pt-BR"; case ELanguage::pt: return "pt-BR";
case ESTTLanguage::zh: return "zh-CN"; case ELanguage::zh: return "zh-CN";
case ESTTLanguage::ja: return "ja-JP"; case ELanguage::ja: return "ja-JP";
case ESTTLanguage::hi: return "hi-IN"; case ELanguage::hi: return "hi-IN";
case ESTTLanguage::it: return "it-IT"; case ELanguage::it: return "it-IT";
case ESTTLanguage::ko: return "ko-KR"; case ELanguage::ko: return "ko-KR";
case ESTTLanguage::nl: return "nl-NL"; case ELanguage::nl: return "nl-NL";
case ESTTLanguage::pl: return "pl-PL"; case ELanguage::pl: return "pl-PL";
case ESTTLanguage::ru: return "ru-RU"; case ELanguage::ru: return "ru-RU";
case ESTTLanguage::sv: return "sv-SE"; case ELanguage::sv: return "sv-SE";
case ESTTLanguage::tr: return "tr-TR"; case ELanguage::tr: return "tr-TR";
case ESTTLanguage::tl: return "fil-PH"; case ELanguage::tl: return "fil-PH";
case ESTTLanguage::bg: return "bg-BG"; case ELanguage::bg: return "bg-BG";
case ESTTLanguage::ro: return "ro-RO"; case ELanguage::ro: return "ro-RO";
case ESTTLanguage::ar: return "ar-SA"; case ELanguage::ar: return "ar-SA";
case ESTTLanguage::cs: return "cs-CZ"; case ELanguage::cs: return "cs-CZ";
case ESTTLanguage::el: return "el-GR"; case ELanguage::el: return "el-GR";
case ESTTLanguage::fi: return "fi-FI"; case ELanguage::fi: return "fi-FI";
case ESTTLanguage::hr: return "hr-HR"; case ELanguage::hr: return "hr-HR";
case ESTTLanguage::ms: return "ms-MY"; case ELanguage::ms: return "ms-MY";
case ESTTLanguage::sk: return "sk-SK"; case ELanguage::sk: return "sk-SK";
case ESTTLanguage::da: return "da-DK"; case ELanguage::da: return "da-DK";
case ESTTLanguage::ta: return "ta-IN"; case ELanguage::ta: return "ta-IN";
case ESTTLanguage::uk: return "uk-UA"; case ELanguage::uk: return "uk-UA";
case ESTTLanguage::hu: return "hu-HU"; case ELanguage::hu: return "hu-HU";
case ESTTLanguage::no: return "nb-NO"; case ELanguage::no: return "nb-NO";
case ESTTLanguage::vi: return "vi-VN"; case ELanguage::vi: return "vi-VN";
case ESTTLanguage::bn: return "bn-IN"; case ELanguage::bn: return "bn-IN";
case ESTTLanguage::th: return "th-TH"; case ELanguage::th: return "th-TH";
case ESTTLanguage::he: return "he-IL"; case ELanguage::he: return "he-IL";
case ESTTLanguage::ka: return "ka-GE"; case ELanguage::ka: return "ka-GE";
case ESTTLanguage::id: return "id-ID"; case ELanguage::id: return "id-ID";
case ESTTLanguage::te: return "te-IN"; case ELanguage::te: return "te-IN";
case ESTTLanguage::gu: return "gu-IN"; case ELanguage::gu: return "gu-IN";
case ESTTLanguage::kn: return "kn-IN"; case ELanguage::kn: return "kn-IN";
case ESTTLanguage::ml: return "ml-IN"; case ELanguage::ml: return "ml-IN";
case ESTTLanguage::mr: return "mr-IN"; case ELanguage::mr: return "mr-IN";
case ESTTLanguage::pa: return "pa-IN"; case ELanguage::pa: return "pa-IN";
default: return "UNDEFINED"; default: return "UNDEFINED";
} }
} }

20
Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Processor/Parakeet/STTParakeetProcessorBase.cpp

@ -36,12 +36,12 @@ void USTTParakeetProcessorBase::InitSTTProcessor(USTTManagerBase* BaseSTTManager
} }
#if WITH_EDITOR #if WITH_EDITOR
if (ParakeetConfig->KeepAliveRule == ESTTKeepAliveRule::Never) if (ParakeetConfig->ParakeetSettings.KeepAliveRule == EKeepAliveRule::Never)
bKeepAlive = false; bKeepAlive = false;
else else
bKeepAlive = true; bKeepAlive = true;
#else #else
if (ParakeetConfig->KeepAliveRule == ESTTKeepAliveRule::Always) if (ParakeetConfig->ParakeetSettings.KeepAliveRule == EKeepAliveRule::Always)
bKeepAlive = true; bKeepAlive = true;
else else
bKeepAlive = false; bKeepAlive = false;
@ -75,8 +75,8 @@ void USTTParakeetProcessorBase::InitSTTProcessor(USTTManagerBase* BaseSTTManager
TSharedRef<FInternetAddr> Addr = Subsystem->CreateInternetAddr(); TSharedRef<FInternetAddr> Addr = Subsystem->CreateInternetAddr();
bool bIsValid = false; bool bIsValid = false;
Addr->SetIp(*ParakeetConfig->Host, bIsValid); Addr->SetIp(*ParakeetConfig->ParakeetSettings.Host, bIsValid);
Addr->SetPort(ParakeetConfig->Port > 0 ? ParakeetConfig->Port : ParakeetDefaultPort); Addr->SetPort(ParakeetConfig->ParakeetSettings.Port > 0 ? ParakeetConfig->ParakeetSettings.Port : ParakeetDefaultPort);
if (!bIsValid) if (!bIsValid)
{ {
STTManager->OnSTTError.Broadcast(TEXT("Invalid address for Parakeet")); STTManager->OnSTTError.Broadcast(TEXT("Invalid address for Parakeet"));
@ -315,9 +315,9 @@ void USTTParakeetProcessorBase::SendConfiguration()
TSharedPtr<FJsonObject> ConfigObj = MakeShared<FJsonObject>(); TSharedPtr<FJsonObject> ConfigObj = MakeShared<FJsonObject>();
TSharedPtr<FJsonObject> ParamsObj = MakeShared<FJsonObject>(); TSharedPtr<FJsonObject> ParamsObj = MakeShared<FJsonObject>();
ParamsObj->SetStringField(TEXT("pretrained_model"), ParakeetConfig->PretrainedModel); ParamsObj->SetStringField(TEXT("pretrained_model"), ParakeetConfig->ParakeetSettings.PretrainedModel);
ParamsObj->SetStringField(TEXT("device"), ParakeetConfig->Device); ParamsObj->SetStringField(TEXT("device"), ParakeetConfig->ParakeetSettings.Device);
ParamsObj->SetNumberField(TEXT("update_interval"), ParakeetConfig->UpdateIntervalSec); ParamsObj->SetNumberField(TEXT("update_interval"), ParakeetConfig->ParakeetSettings.UpdateIntervalSec);
ConfigObj->SetStringField(TEXT("type"), TEXT("config")); ConfigObj->SetStringField(TEXT("type"), TEXT("config"));
ConfigObj->SetObjectField(TEXT("params"), ParamsObj); ConfigObj->SetObjectField(TEXT("params"), ParamsObj);
@ -429,9 +429,9 @@ bool USTTParakeetProcessorBase::StartParakeetProcess()
} }
const FString Args = FString::Printf(TEXT("\"%s\" serve-config --host %s --port %d"), const FString Args = FString::Printf(TEXT("\"%s\" serve-config --host %s --port %d"),
*ParakeetConfig->PythonPath, *ParakeetConfig->ParakeetSettings.PythonPath,
*ParakeetConfig->Host, *ParakeetConfig->ParakeetSettings.Host,
ParakeetConfig->Port > 0 ? ParakeetConfig->Port : ParakeetDefaultPort); ParakeetConfig->ParakeetSettings.Port > 0 ? ParakeetConfig->ParakeetSettings.Port : ParakeetDefaultPort);
uint32 ProcId = 0; uint32 ProcId = 0;

16
Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Processor/Whisper/STTProcessorWhisper.cpp

@ -51,7 +51,7 @@ void USTTProcessorWhisper::InitSTTProcessor(USTTManagerBase* BaseSTTManager, UST
return; return;
} }
if (WhisperProcessorConfig->OpenAI_API_Key.IsEmpty()) { if (WhisperProcessorConfig->WhisperSettings.OpenAI_API_Key.IsEmpty()) {
if (IsValid(STTManager)) if (IsValid(STTManager))
STTManager->OnSTTError.Broadcast(TEXT("OpenAI API Key not set. Needs to be done before initializing modules.")); STTManager->OnSTTError.Broadcast(TEXT("OpenAI API Key not set. Needs to be done before initializing modules."));
return; return;
@ -61,7 +61,7 @@ void USTTProcessorWhisper::InitSTTProcessor(USTTManagerBase* BaseSTTManager, UST
PerformHealthCheck(); PerformHealthCheck();
STTManager->OnSTTLog.Broadcast(FString::Printf(TEXT("STTProcessor OpenAI %s initialized successfully."), *TranscribeModelEnumToString(WhisperProcessorConfig->Model))); STTManager->OnSTTLog.Broadcast(FString::Printf(TEXT("STTProcessor OpenAI %s initialized successfully."), *TranscribeModelEnumToString(WhisperProcessorConfig->WhisperSettings.Model)));
} }
void USTTProcessorWhisper::ClearSTTProcessor() void USTTProcessorWhisper::ClearSTTProcessor()
@ -114,7 +114,7 @@ void USTTProcessorWhisper::StartTranscriptionFromBuffer()
{ {
const float Frames = static_cast<float>(PCMDataCopy.Num()) / static_cast<float>(AudioInfoCopy.NumChannels); const float Frames = static_cast<float>(PCMDataCopy.Num()) / static_cast<float>(AudioInfoCopy.NumChannels);
const float DurationSeconds = Frames / static_cast<float>(AudioInfoCopy.SampleRate); const float DurationSeconds = Frames / static_cast<float>(AudioInfoCopy.SampleRate);
if (DurationSeconds < WhisperProcessorConfig->MinDuration) if (DurationSeconds < WhisperProcessorConfig->WhisperSettings.MinDuration)
{ {
return; return;
} }
@ -198,9 +198,9 @@ void USTTProcessorWhisper::BuildMultipartBody(const TArray<uint8>& WavData, cons
FString BoundaryLine = FString::Printf(TEXT("--%s\r\n"), *Boundary); FString BoundaryLine = FString::Printf(TEXT("--%s\r\n"), *Boundary);
AppendStringToBody(OutBody, BoundaryLine); AppendStringToBody(OutBody, BoundaryLine);
AppendStringToBody(OutBody, TEXT("Content-Disposition: form-data; name=\"model\"\r\n\r\n")); AppendStringToBody(OutBody, TEXT("Content-Disposition: form-data; name=\"model\"\r\n\r\n"));
AppendStringToBody(OutBody, TranscribeModelEnumToString(WhisperProcessorConfig->Model) + TEXT("\r\n")); AppendStringToBody(OutBody, TranscribeModelEnumToString(WhisperProcessorConfig->WhisperSettings.Model) + TEXT("\r\n"));
if (WhisperProcessorConfig->Model == EOpenAITranscriptionModel::Whisper1) if (WhisperProcessorConfig->WhisperSettings.Model == EOpenAITranscriptionModel::Whisper1)
{ {
BoundaryLine = FString::Printf(TEXT("--%s\r\n"), *Boundary); BoundaryLine = FString::Printf(TEXT("--%s\r\n"), *Boundary);
AppendStringToBody(OutBody, BoundaryLine); AppendStringToBody(OutBody, BoundaryLine);
@ -233,7 +233,7 @@ void USTTProcessorWhisper::NormalizeWhisperURL()
if (!WhisperProcessorConfig) if (!WhisperProcessorConfig)
return; return;
NormalizedWhisperURL = WhisperProcessorConfig->WhisperURL; NormalizedWhisperURL = WhisperProcessorConfig->WhisperSettings.WhisperURL;
if (!NormalizedWhisperURL.StartsWith(TEXT("http://")) && !NormalizedWhisperURL.StartsWith(TEXT("https://"))) if (!NormalizedWhisperURL.StartsWith(TEXT("http://")) && !NormalizedWhisperURL.StartsWith(TEXT("https://")))
{ {
NormalizedWhisperURL = FString::Printf(TEXT("https://%s"), *NormalizedWhisperURL); NormalizedWhisperURL = FString::Printf(TEXT("https://%s"), *NormalizedWhisperURL);
@ -252,7 +252,7 @@ void USTTProcessorWhisper::PerformHealthCheck()
TSharedRef<IHttpRequest, ESPMode::ThreadSafe> Request = HttpModule.CreateRequest(); TSharedRef<IHttpRequest, ESPMode::ThreadSafe> Request = HttpModule.CreateRequest();
Request->SetURL(NormalizedWhisperURL); Request->SetURL(NormalizedWhisperURL);
Request->SetVerb(TEXT("GET")); Request->SetVerb(TEXT("GET"));
FString AuthHeader = FString::Printf(TEXT("Bearer %s"), *WhisperProcessorConfig->OpenAI_API_Key); FString AuthHeader = FString::Printf(TEXT("Bearer %s"), *WhisperProcessorConfig->WhisperSettings.OpenAI_API_Key);
Request->SetHeader(TEXT("Authorization"), AuthHeader); Request->SetHeader(TEXT("Authorization"), AuthHeader);
Request->OnProcessRequestComplete().BindLambda([ Request->OnProcessRequestComplete().BindLambda([
WeakManager = TWeakObjectPtr<USTTManagerBase>(STTManager)](FHttpRequestPtr Req, FHttpResponsePtr Resp, bool bWasSuccessful) WeakManager = TWeakObjectPtr<USTTManagerBase>(STTManager)](FHttpRequestPtr Req, FHttpResponsePtr Resp, bool bWasSuccessful)
@ -292,7 +292,7 @@ void USTTProcessorWhisper::SendWhisperRequest(TArray<uint8>&& WavData)
Request->SetURL(NormalizedWhisperURL); Request->SetURL(NormalizedWhisperURL);
Request->SetVerb(TEXT("POST")); Request->SetVerb(TEXT("POST"));
FString AuthHeader = FString::Printf(TEXT("Bearer %s"), *WhisperProcessorConfig->OpenAI_API_Key); FString AuthHeader = FString::Printf(TEXT("Bearer %s"), *WhisperProcessorConfig->WhisperSettings.OpenAI_API_Key);
Request->SetHeader(TEXT("Authorization"), AuthHeader); Request->SetHeader(TEXT("Authorization"), AuthHeader);
Request->SetHeader(TEXT("Accept"), TEXT("application/json")); Request->SetHeader(TEXT("Accept"), TEXT("application/json"));

2
Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/STTManagerBase.cpp

@ -175,7 +175,7 @@ bool USTTManagerBase::IsBlocked()
return (CurrentSpeechState==ESTTTalkingState::BLOCKED); return (CurrentSpeechState==ESTTTalkingState::BLOCKED);
} }
void USTTManagerBase::SetLanguage(TArray<ESTTLanguage> NewLanguages) void USTTManagerBase::SetSTTLanguage(TArray<ELanguage> NewLanguages)
{ {
if (!IsValid(ProcessorConfig)) if (!IsValid(ProcessorConfig))
return; return;

16
Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Processor/Azure/STTAzureProcessorConfig.h

@ -6,6 +6,18 @@
#include "Processor/STTBaseProcessorConfig.h" #include "Processor/STTBaseProcessorConfig.h"
#include "STTAzureProcessorConfig.generated.h" #include "STTAzureProcessorConfig.generated.h"
USTRUCT(BlueprintType)
struct FSTTAzureSettings
{
GENERATED_BODY()
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreSTT|Azure", meta = (ExposeOnSpawn = "true"))
FString AzureAPIKey = "";
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreSTT|Azure", meta = (ExposeOnSpawn = "true"))
FString AzureRegion = "germanywestcentral";
};
/** /**
* *
*/ */
@ -19,7 +31,5 @@ public:
USTTAzureProcessorConfig(const FObjectInitializer& ObjectInitializer); USTTAzureProcessorConfig(const FObjectInitializer& ObjectInitializer);
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreSTT|Azure", meta = (ExposeOnSpawn = "true")) UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreSTT|Azure", meta = (ExposeOnSpawn = "true"))
FString AzureAPIKey = ""; FSTTAzureSettings AzureSettings;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreSTT|Azure", meta = (ExposeOnSpawn = "true"))
FString AzureRegion = "germanywestcentral";
}; };

4
Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Processor/Azure/STTProcessorAzure.h

@ -32,7 +32,7 @@ class AVATARCORE_STT_API USTTProcessorAzure : public USTTProcessorBase
public: public:
UFUNCTION(BlueprintCallable, Category = STTManager) UFUNCTION(BlueprintCallable, Category = STTManager)
virtual void ChangeAzureLanguage(TArray<ESTTLanguage> InLanguages); virtual void ChangeAzureLanguage(TArray<ELanguage> InLanguages);
private: private:
@ -62,7 +62,7 @@ public:
void OnAzureError(FString Error); void OnAzureError(FString Error);
UFUNCTION(BlueprintPure, Category = STTManager) UFUNCTION(BlueprintPure, Category = STTManager)
FString AzureEnumToString(ESTTLanguage Language); FString AzureEnumToString(ELanguage Language);
USTTAzureProcessorConfig* AzureProcessorConfig; USTTAzureProcessorConfig* AzureProcessorConfig;
}; };

30
Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Processor/Parakeet/STTParakeetProcessorConfig.h

@ -6,18 +6,11 @@
#include "Processor/STTBaseProcessorConfig.h" #include "Processor/STTBaseProcessorConfig.h"
#include "STTParakeetProcessorConfig.generated.h" #include "STTParakeetProcessorConfig.generated.h"
/** USTRUCT(BlueprintType)
* Configuration for the Parakeet STT Processor (NVIDIA NeMo ASR). struct FSTTParakeetSettings
*/
UCLASS(Blueprintable, BlueprintType)
class AVATARCORE_STT_API USTTParakeetProcessorConfig : public USTTBaseProcessorConfig
{ {
GENERATED_BODY() GENERATED_BODY()
public:
USTTParakeetProcessorConfig(const FObjectInitializer& ObjectInitializer);
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreSTT|Parakeet", meta = (ExposeOnSpawn = "true")) UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreSTT|Parakeet", meta = (ExposeOnSpawn = "true"))
FString PythonPath = "python"; FString PythonPath = "python";
@ -31,7 +24,7 @@ public:
int32 Port = 40200; int32 Port = 40200;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreSTT|Parakeet", meta = (ExposeOnSpawn = "true")) UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreSTT|Parakeet", meta = (ExposeOnSpawn = "true"))
ESTTKeepAliveRule KeepAliveRule = ESTTKeepAliveRule::EditorOnly; EKeepAliveRule KeepAliveRule = EKeepAliveRule::EditorOnly;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreSTT|Parakeet", meta = (ExposeOnSpawn = "true")) UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreSTT|Parakeet", meta = (ExposeOnSpawn = "true"))
FString Device = "cuda:0"; FString Device = "cuda:0";
@ -39,4 +32,21 @@ public:
// How often (seconds) the Python server produces intermediate transcription updates // How often (seconds) the Python server produces intermediate transcription updates
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreSTT|Parakeet", meta = (ExposeOnSpawn = "true", ClampMin = "0.1", ClampMax = "5.0")) UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreSTT|Parakeet", meta = (ExposeOnSpawn = "true", ClampMin = "0.1", ClampMax = "5.0"))
float UpdateIntervalSec = 0.5f; float UpdateIntervalSec = 0.5f;
};
/**
* Configuration for the Parakeet STT Processor (NVIDIA NeMo ASR).
*/
UCLASS(Blueprintable, BlueprintType)
class AVATARCORE_STT_API USTTParakeetProcessorConfig : public USTTBaseProcessorConfig
{
GENERATED_BODY()
public:
USTTParakeetProcessorConfig(const FObjectInitializer& ObjectInitializer);
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreSTT|Parakeet", meta = (ExposeOnSpawn = "true", ClampMin = "0.1", ClampMax = "5.0"))
FSTTParakeetSettings ParakeetSettings;
}; };

7
Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Processor/STTBaseProcessorConfig.h

@ -7,13 +7,6 @@
#include "STTStructs.h" #include "STTStructs.h"
#include "STTBaseProcessorConfig.generated.h" #include "STTBaseProcessorConfig.generated.h"
UENUM(BlueprintType)
enum class ESTTKeepAliveRule : uint8 {
EditorOnly UMETA(DisplayName = "Only keep STT Module open in Editor Mode"),
Never UMETA(DisplayName = "Never keep STT Module alive."),
Always UMETA(DisplayName = "Keep STT Module alive in Editor and Shipping mode.")
};
class USTTProcessorBase; class USTTProcessorBase;
/** /**

25
Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Processor/Whisper/STTWhisperProcessorConfig.h

@ -14,6 +14,23 @@ enum class EOpenAITranscriptionModel : uint8
Transcribe4o UMETA(DisplayName = "4o Transcribe") Transcribe4o UMETA(DisplayName = "4o Transcribe")
}; };
USTRUCT(BlueprintType)
struct FSTTWhisperSettings
{
GENERATED_BODY()
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreSTT|Whisper", meta = (ExposeOnSpawn = "true"))
FString OpenAI_API_Key = "";
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreSTT|Whisper", meta = (ExposeOnSpawn = "true"))
FString WhisperURL = "api.openai.com/v1/audio/transcriptions";
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreSTT|Whisper", meta = (ExposeOnSpawn = "true"))
EOpenAITranscriptionModel Model = EOpenAITranscriptionModel::Transcribe4o;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreSTT|Whisper", meta = (ExposeOnSpawn = "true"))
float MinDuration = 0.75f;
};
/** /**
* *
*/ */
@ -27,12 +44,6 @@ public:
USTTWhisperProcessorConfig(const FObjectInitializer& ObjectInitializer); USTTWhisperProcessorConfig(const FObjectInitializer& ObjectInitializer);
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreSTT|Whisper", meta = (ExposeOnSpawn = "true")) UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreSTT|Whisper", meta = (ExposeOnSpawn = "true"))
FString OpenAI_API_Key = ""; FSTTWhisperSettings WhisperSettings;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreSTT|Whisper", meta = (ExposeOnSpawn = "true"))
FString WhisperURL = "api.openai.com/v1/audio/transcriptions";
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreSTT|Whisper", meta = (ExposeOnSpawn = "true"))
EOpenAITranscriptionModel Model = EOpenAITranscriptionModel::Transcribe4o;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreSTT|Whisper", meta = (ExposeOnSpawn = "true"))
float MinDuration = 0.75f;
}; };

2
Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/STTManagerBase.h

@ -99,7 +99,7 @@ public:
bool IsBlocked(); bool IsBlocked();
UFUNCTION(BlueprintCallable, Category = "AvatarCoreSTT") UFUNCTION(BlueprintCallable, Category = "AvatarCoreSTT")
void SetLanguage(TArray<ESTTLanguage> NewLanguages); void SetSTTLanguage(TArray<ELanguage> NewLanguages);
UFUNCTION(BlueprintCallable, Category = "AvatarCoreSTT") UFUNCTION(BlueprintCallable, Category = "AvatarCoreSTT")
void AddSpecialWord(FString NewWord); void AddSpecialWord(FString NewWord);

51
Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/STTStructs.h

@ -1,6 +1,7 @@
#pragma once #pragma once
#include "CoreMinimal.h" #include "CoreMinimal.h"
#include "AvatarCoreSharedEnums.h"
#include "STTStructs.generated.h" #include "STTStructs.generated.h"
USTRUCT(BlueprintType) USTRUCT(BlueprintType)
@ -70,54 +71,6 @@ enum class ESpeexDSPState : uint8
SPEEXPREPROCESS_SET_AGC_TARGET = 46 UMETA(DisplayName = "preprocessor Automatic Gain Control level (int32)") SPEEXPREPROCESS_SET_AGC_TARGET = 46 UMETA(DisplayName = "preprocessor Automatic Gain Control level (int32)")
}; };
UENUM(BlueprintType)
enum class ESTTLanguage : uint8
{
NONE UMETA(DisplayName = "Unset"),
en UMETA(DisplayName = "English"),
fr UMETA(DisplayName = "French"),
de UMETA(DisplayName = "German"),
es UMETA(DisplayName = "Spanish"),
pt UMETA(DisplayName = "Portuguese"),
zh UMETA(DisplayName = "Chinese"),
ja UMETA(DisplayName = "Japanese"),
hi UMETA(DisplayName = "Hindi"),
it UMETA(DisplayName = "Italian"),
ko UMETA(DisplayName = "Korean"),
nl UMETA(DisplayName = "Dutch"),
pl UMETA(DisplayName = "Polish"),
ru UMETA(DisplayName = "Russian"),
sv UMETA(DisplayName = "Swedish"),
tr UMETA(DisplayName = "Turkish"),
tl UMETA(DisplayName = "Filipino"),
bg UMETA(DisplayName = "Bulgarian"),
ro UMETA(DisplayName = "Romanian"),
ar UMETA(DisplayName = "Arabic"),
cs UMETA(DisplayName = "Czech"),
el UMETA(DisplayName = "Greek"),
fi UMETA(DisplayName = "Finnish"),
hr UMETA(DisplayName = "Croatian"),
ms UMETA(DisplayName = "Malay"),
sk UMETA(DisplayName = "Slovak"),
da UMETA(DisplayName = "Danish"),
ta UMETA(DisplayName = "Tamil"),
uk UMETA(DisplayName = "Ukrainian"),
hu UMETA(DisplayName = "Hungarian"),
no UMETA(DisplayName = "Norwegian"),
vi UMETA(DisplayName = "Vietnamese"),
bn UMETA(DisplayName = "Bengali"),
th UMETA(DisplayName = "Thai"),
he UMETA(DisplayName = "Hebrew"),
ka UMETA(DisplayName = "Georgian"),
id UMETA(DisplayName = "Indonesian"),
te UMETA(DisplayName = "Telugu"),
gu UMETA(DisplayName = "Gujarati"),
kn UMETA(DisplayName = "Kannada"),
ml UMETA(DisplayName = "Malayalam"),
mr UMETA(DisplayName = "Marathi"),
pa UMETA(DisplayName = "Punjabi"),
};
USTRUCT(BlueprintType) USTRUCT(BlueprintType)
struct FWebRTCSettings struct FWebRTCSettings
{ {
@ -262,7 +215,7 @@ struct FSTTBaseSettings
UPROPERTY(EditAnywhere, BlueprintReadWrite, meta = (ToolTip = "Settings of the SpeexDSP Module", Category = "STT|Base")) UPROPERTY(EditAnywhere, BlueprintReadWrite, meta = (ToolTip = "Settings of the SpeexDSP Module", Category = "STT|Base"))
FSpeexDSPSettings SpeexDSPSettings; FSpeexDSPSettings SpeexDSPSettings;
UPROPERTY(EditAnywhere, BlueprintReadWrite, meta = (ToolTip = "All languages the STT module should understand simultaneously.", Category = "STT|Base")) UPROPERTY(EditAnywhere, BlueprintReadWrite, meta = (ToolTip = "All languages the STT module should understand simultaneously.", Category = "STT|Base"))
TArray<ESTTLanguage> STTLanguages = { ESTTLanguage::de, ESTTLanguage::en }; TArray<ELanguage> STTLanguages = { ELanguage::de, ELanguage::en };
UPROPERTY(EditAnywhere, BlueprintReadWrite, meta = (ToolTip = "Transcriptions to always change to another word.", Category = "STT|Base")) UPROPERTY(EditAnywhere, BlueprintReadWrite, meta = (ToolTip = "Transcriptions to always change to another word.", Category = "STT|Base"))
TArray<FSTTWordReplacement> STTReplacements; TArray<FSTTWordReplacement> STTReplacements;
UPROPERTY(EditAnywhere, BlueprintReadWrite, meta = (ToolTip = "Special words that the transcription service needs to know (e.g. b.ReX or Bruce-B).", Category = "STT|Base")) UPROPERTY(EditAnywhere, BlueprintReadWrite, meta = (ToolTip = "Special words that the transcription service needs to know (e.g. b.ReX or Bruce-B).", Category = "STT|Base"))

24
Unreal/Plugins/AvatarCore_Shared/AvatarCore_Shared.uplugin

@ -0,0 +1,24 @@
{
"FileVersion": 3,
"Version": 1,
"VersionName": "1.0",
"FriendlyName": "AvatarCore_Shared",
"Description": "Shared assets, structs and enums. In theory we want to be AI, TTS and STT to be separate.",
"Category": "Other",
"CreatedBy": "b.ReX GmbH",
"CreatedByURL": "",
"DocsURL": "",
"MarketplaceURL": "",
"SupportURL": "",
"CanContainContent": true,
"IsBetaVersion": false,
"IsExperimentalVersion": false,
"Installed": false,
"Modules": [
{
"Name": "AvatarCore_Shared",
"Type": "Runtime",
"LoadingPhase": "Default"
}
]
}

BIN
Unreal/Plugins/AvatarCore_Shared/Resources/Icon128.png (Stored with Git LFS)

Binary file not shown.

53
Unreal/Plugins/AvatarCore_Shared/Source/AvatarCore_Shared/AvatarCore_Shared.Build.cs

@ -0,0 +1,53 @@
// Copyright Epic Games, Inc. All Rights Reserved.
using UnrealBuildTool;
public class AvatarCore_Shared : ModuleRules
{
public AvatarCore_Shared(ReadOnlyTargetRules Target) : base(Target)
{
PCHUsage = ModuleRules.PCHUsageMode.UseExplicitOrSharedPCHs;
PublicIncludePaths.AddRange(
new string[] {
// ... add public include paths required here ...
}
);
PrivateIncludePaths.AddRange(
new string[] {
// ... add other private include paths required here ...
}
);
PublicDependencyModuleNames.AddRange(
new string[]
{
"Core",
// ... add other public dependencies that you statically link with here ...
}
);
PrivateDependencyModuleNames.AddRange(
new string[]
{
"CoreUObject",
"Engine",
"Slate",
"SlateCore",
// ... add private dependencies that you statically link with here ...
}
);
DynamicallyLoadedModuleNames.AddRange(
new string[]
{
// ... add any modules that your module loads dynamically here ...
}
);
}
}

20
Unreal/Plugins/AvatarCore_Shared/Source/AvatarCore_Shared/Private/AvatarCore_Shared.cpp

@ -0,0 +1,20 @@
// Copyright Epic Games, Inc. All Rights Reserved.
#include "AvatarCore_Shared.h"
#define LOCTEXT_NAMESPACE "FAvatarCore_SharedModule"
void FAvatarCore_SharedModule::StartupModule()
{
// This code will execute after your module is loaded into memory; the exact timing is specified in the .uplugin file per-module
}
void FAvatarCore_SharedModule::ShutdownModule()
{
// This function may be called during shutdown to clean up your module. For modules that support dynamic reloading,
// we call this function before unloading the module.
}
#undef LOCTEXT_NAMESPACE
IMPLEMENT_MODULE(FAvatarCore_SharedModule, AvatarCore_Shared)

27
Unreal/Plugins/AvatarCore_Shared/Source/AvatarCore_Shared/Private/FL_AvatarCoreShared.cpp

@ -0,0 +1,27 @@
// Fill out your copyright notice in the Description page of Project Settings.
#include "FL_AvatarCoreShared.h"
ELanguage UFL_AvatarCoreShared::StringToLanguage(const FString& InString)
{
const UEnum* EnumPtr = StaticEnum<ELanguage>();
if (!EnumPtr)
{
return ELanguage::NONE;
}
FString Normalized = InString.ToLower();
int64 Value = EnumPtr->GetValueByNameString(Normalized);
return Value != INDEX_NONE
? static_cast<ELanguage>(Value)
: ELanguage::NONE;
}
FString UFL_AvatarCoreShared::LanguageToString(ELanguage Language)
{
return StaticEnum<ELanguage>()->GetNameStringByValue((int64)Language);
}

62
Unreal/Plugins/AvatarCore_Shared/Source/AvatarCore_Shared/Public/AvatarCoreSharedEnums.h

@ -0,0 +1,62 @@
// Fill out your copyright notice in the Description page of Project Settings.
#pragma once
#include "CoreMinimal.h"
#include "UObject/NoExportTypes.h"
#include "AvatarCoreSharedEnums.generated.h"
UENUM(BlueprintType)
enum class ELanguage : uint8 {
NONE UMETA(DisplayName = "Unset"),
en UMETA(DisplayName = "English"),
fr UMETA(DisplayName = "French"),
de UMETA(DisplayName = "German"),
es UMETA(DisplayName = "Spanish"),
pt UMETA(DisplayName = "Portuguese"),
zh UMETA(DisplayName = "Chinese"),
ja UMETA(DisplayName = "Japanese"),
hi UMETA(DisplayName = "Hindi"),
it UMETA(DisplayName = "Italian"),
ko UMETA(DisplayName = "Korean"),
nl UMETA(DisplayName = "Dutch"),
pl UMETA(DisplayName = "Polish"),
ru UMETA(DisplayName = "Russian"),
sv UMETA(DisplayName = "Swedish"),
tr UMETA(DisplayName = "Turkish"),
tl UMETA(DisplayName = "Filipino"),
bg UMETA(DisplayName = "Bulgarian"),
ro UMETA(DisplayName = "Romanian"),
ar UMETA(DisplayName = "Arabic"),
cs UMETA(DisplayName = "Czech"),
el UMETA(DisplayName = "Greek"),
fi UMETA(DisplayName = "Finnish"),
hr UMETA(DisplayName = "Croatian"),
ms UMETA(DisplayName = "Malay"),
sk UMETA(DisplayName = "Slovak"),
da UMETA(DisplayName = "Danish"),
ta UMETA(DisplayName = "Tamil"),
uk UMETA(DisplayName = "Ukrainian"),
hu UMETA(DisplayName = "Hungarian"),
no UMETA(DisplayName = "Norwegian"),
vi UMETA(DisplayName = "Vietnamese"),
bn UMETA(DisplayName = "Bengali"),
th UMETA(DisplayName = "Thai"),
he UMETA(DisplayName = "Hebrew"),
ka UMETA(DisplayName = "Georgian"),
id UMETA(DisplayName = "Indonesian"),
te UMETA(DisplayName = "Telugu"),
gu UMETA(DisplayName = "Gujarati"),
kn UMETA(DisplayName = "Kannada"),
ml UMETA(DisplayName = "Malayalam"),
mr UMETA(DisplayName = "Marathi"),
pa UMETA(DisplayName = "Punjabi"),
};
UENUM(BlueprintType)
enum class EKeepAliveRule : uint8 {
EditorOnly UMETA(DisplayName = "Only keep Module open in Editor Mode"),
Never UMETA(DisplayName = "Never keep Module alive."),
Always UMETA(DisplayName = "Keep Module alive in Editor and Shipping mode.")
};

14
Unreal/Plugins/AvatarCore_Shared/Source/AvatarCore_Shared/Public/AvatarCore_Shared.h

@ -0,0 +1,14 @@
// Copyright Epic Games, Inc. All Rights Reserved.
#pragma once
#include "Modules/ModuleManager.h"
class FAvatarCore_SharedModule : public IModuleInterface
{
public:
/** IModuleInterface implementation */
virtual void StartupModule() override;
virtual void ShutdownModule() override;
};

27
Unreal/Plugins/AvatarCore_Shared/Source/AvatarCore_Shared/Public/FL_AvatarCoreShared.h

@ -0,0 +1,27 @@
// Fill out your copyright notice in the Description page of Project Settings.
#pragma once
#include "CoreMinimal.h"
#include "Kismet/BlueprintFunctionLibrary.h"
#include "AvatarCoreSharedEnums.h"
#include "FL_AvatarCoreShared.generated.h"
/**
*
*/
UCLASS()
class AVATARCORE_SHARED_API UFL_AvatarCoreShared : public UBlueprintFunctionLibrary
{
GENERATED_BODY()
public:
// Returns the ISO 639-1 code string for the given language enum (e.g. ELanguage::de "de").
// Derived from the enum value name via UE reflection, so it stays in sync automatically.
UFUNCTION(BlueprintCallable, Category = "AvatarCoreShared")
static FString LanguageToString(ELanguage Language);
// Returns the ELanguage for the given ISO 639-1 code string
UFUNCTION(BlueprintCallable, Category = "AvatarCoreShared")
static ELanguage StringToLanguage(const FString& InString);
};

4
Unreal/Plugins/AvatarCore_TTS/AvatarCore_TTS.uplugin

@ -25,6 +25,10 @@
{ {
"Name": "BTools", "Name": "BTools",
"Enabled": true "Enabled": true
},
{
"Name": "AvatarCore_Shared",
"Enabled": true
} }
] ]
} }

9
Unreal/Plugins/AvatarCore_TTS/Source/.claude/settings.local.json

@ -0,0 +1,9 @@
{
"permissions": {
"allow": [
"Bash(Get-ChildItem -Path \"c:\\\\Gitprojects\\\\SchaniConcierge\\\\Unreal\\\\Plugins\\\\AvatarCore_TTS\\\\Source\" -Recurse -Filter \"CartesiaTTSManager.h\")",
"Bash(Select-Object -ExpandProperty FullName)",
"Bash(find c:\\\\\\\\Gitprojects\\\\\\\\SchaniConcierge\\\\\\\\Unreal\\\\\\\\Plugins\\\\\\\\AvatarCore_TTS\\\\\\\\Source -name \"CartesiaTTSManager.h\")"
]
}
}

1
Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/AvatarCore_TTS.Build.cs

@ -35,6 +35,7 @@ public class AvatarCore_TTS : ModuleRules
"AudioMixer", "AudioMixer",
"AudioExtensions", "AudioExtensions",
"WebRTC", "WebRTC",
"AvatarCore_Shared",
// ... add other public dependencies that you statically link with here ... // ... add other public dependencies that you statically link with here ...
} }
); );

68
Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Private/Cartesia/CartesiaTTSManager.cpp

@ -1,8 +1,8 @@
// Fill out your copyright notice in the Description page of Project Settings. // Fill out your copyright notice in the Description page of Project Settings.
#include "Cartesia/CartesiaTTSManager.h" #include "Cartesia/CartesiaTTSManager.h"
#include "FL_AvatarCoreShared.h"
#include "WebSocketsModule.h" #include "WebSocketsModule.h"
#include "IWebSocket.h" #include "IWebSocket.h"
#include "Async/Async.h" #include "Async/Async.h"
@ -168,7 +168,7 @@ void UCartesiaTTSManager::SendTranscriptMessage(int32 TaskID, const FString& Tra
TSharedPtr<FJsonObject> Obj = MakeShared<FJsonObject>(); TSharedPtr<FJsonObject> Obj = MakeShared<FJsonObject>();
Obj->SetStringField(TEXT("model_id"), CartesiaTTSConfig->CartesiaTTSSettings.CartesiaModelId); Obj->SetStringField(TEXT("model_id"), CartesiaTTSConfig->CartesiaTTSSettings.CartesiaModelId);
Obj->SetStringField(TEXT("transcript"), Transcript); Obj->SetStringField(TEXT("transcript"), Transcript);
Obj->SetStringField(TEXT("language"), TTSLanguageToString(CartesiaTTSConfig->GlobalTTSSettings.Language)); Obj->SetStringField(TEXT("language"), UFL_AvatarCoreShared::LanguageToString(CartesiaTTSConfig->GlobalTTSSettings.Language));
Obj->SetStringField(TEXT("context_id"), GetOrCreateContextId(TaskID)); Obj->SetStringField(TEXT("context_id"), GetOrCreateContextId(TaskID));
Obj->SetBoolField(TEXT("continue"), bContinue); Obj->SetBoolField(TEXT("continue"), bContinue);
@ -183,8 +183,8 @@ void UCartesiaTTSManager::SendTranscriptMessage(int32 TaskID, const FString& Tra
OutputObj->SetNumberField(TEXT("sample_rate"), (TTSConfig ? TTSConfig->GlobalTTSSettings.AudioSampleRate : 24000)); OutputObj->SetNumberField(TEXT("sample_rate"), (TTSConfig ? TTSConfig->GlobalTTSSettings.AudioSampleRate : 24000));
Obj->SetObjectField(TEXT("output_format"), OutputObj); Obj->SetObjectField(TEXT("output_format"), OutputObj);
Obj->SetBoolField(TEXT("add_timestamps"), false); Obj->SetBoolField(TEXT("add_timestamps"), true);
SendJsonForTask(TaskID, Obj); SendJsonForTask(TaskID, Obj);
{ {
@ -217,7 +217,7 @@ void UCartesiaTTSManager::SendFlushMessage(int32 TaskID)
TSharedPtr<FJsonObject> Obj = MakeShared<FJsonObject>(); TSharedPtr<FJsonObject> Obj = MakeShared<FJsonObject>();
Obj->SetStringField(TEXT("model_id"), CartesiaTTSConfig->CartesiaTTSSettings.CartesiaModelId); Obj->SetStringField(TEXT("model_id"), CartesiaTTSConfig->CartesiaTTSSettings.CartesiaModelId);
Obj->SetStringField(TEXT("transcript"), TEXT("")); Obj->SetStringField(TEXT("transcript"), TEXT(""));
Obj->SetStringField(TEXT("language"), TTSLanguageToString(CartesiaTTSConfig->GlobalTTSSettings.Language)); Obj->SetStringField(TEXT("language"), UFL_AvatarCoreShared::LanguageToString(CartesiaTTSConfig->GlobalTTSSettings.Language));
Obj->SetStringField(TEXT("context_id"), GetOrCreateContextId(TaskID)); Obj->SetStringField(TEXT("context_id"), GetOrCreateContextId(TaskID));
Obj->SetBoolField(TEXT("continue"), true); Obj->SetBoolField(TEXT("continue"), true);
Obj->SetBoolField(TEXT("flush"), true); Obj->SetBoolField(TEXT("flush"), true);
@ -233,7 +233,7 @@ void UCartesiaTTSManager::SendFlushMessage(int32 TaskID)
OutputObj->SetNumberField(TEXT("sample_rate"), (TTSConfig ? TTSConfig->GlobalTTSSettings.AudioSampleRate : 24000)); OutputObj->SetNumberField(TEXT("sample_rate"), (TTSConfig ? TTSConfig->GlobalTTSSettings.AudioSampleRate : 24000));
Obj->SetObjectField(TEXT("output_format"), OutputObj); Obj->SetObjectField(TEXT("output_format"), OutputObj);
Obj->SetBoolField(TEXT("add_timestamps"), false); Obj->SetBoolField(TEXT("add_timestamps"), true);
SendJsonForTask(TaskID, Obj); SendJsonForTask(TaskID, Obj);
} }
@ -292,8 +292,8 @@ void UCartesiaTTSManager::StartStreamingGeneration(int32 TaskID, const FString&
Self->TTSLog(FString::Printf(TEXT("[Cartesia] WS connected for task %d"), TaskID)); Self->TTSLog(FString::Printf(TEXT("[Cartesia] WS connected for task %d"), TaskID));
}); });
WS->OnMessage().AddLambda([WeakThis, TaskID](const FString& Message) WS->OnMessage().AddLambda([WeakThis, TaskID](const FString &Message)
{ {
if (!WeakThis.IsValid()) return; if (!WeakThis.IsValid()) return;
UCartesiaTTSManager* Self = WeakThis.Get(); UCartesiaTTSManager* Self = WeakThis.Get();
//Self->TTSLog(FString::Printf(TEXT("[Cartesia][%d] <- %s"), TaskID, *Message)); //Self->TTSLog(FString::Printf(TEXT("[Cartesia][%d] <- %s"), TaskID, *Message));
@ -330,6 +330,53 @@ void UCartesiaTTSManager::StartStreamingGeneration(int32 TaskID, const FString&
return; return;
} }
if (Type.Equals(TEXT("timestamps"), ESearchCase::IgnoreCase))
{
const TSharedPtr<FJsonObject>* WTSObj;
if (Obj->TryGetObjectField(TEXT("word_timestamps"), WTSObj))
{
const TArray<TSharedPtr<FJsonValue>>* WordArr;
const TArray<TSharedPtr<FJsonValue>>* StartArr;
const TArray<TSharedPtr<FJsonValue>>* EndArr;
if ((*WTSObj)->TryGetArrayField(TEXT("words"), WordArr) &&
(*WTSObj)->TryGetArrayField(TEXT("start"), StartArr) &&
(*WTSObj)->TryGetArrayField(TEXT("end"), EndArr))
{
const int32 Count = FMath::Min3(WordArr->Num(), StartArr->Num(), EndArr->Num());
TArray<FWordTimestamp> Timestamps;
Timestamps.Reserve(Count);
for (int32 i = 0; i < Count; ++i)
{
FWordTimestamp WTS;
WTS.Word = (*WordArr)[i]->AsString();
WTS.Start = (float)(*StartArr)[i]->AsNumber();
WTS.End = (float)(*EndArr)[i]->AsNumber();
Timestamps.Add(WTS);
}
AsyncTask(ENamedThreads::GameThread, [Self, TaskID, TS = MoveTemp(Timestamps)]() mutable
{
FScopeLock Lock(&Self->TaskQueueCriticalSection);
for (FTTSTask& Task : Self->ActiveTasks)
{
if (Task.TaskID == TaskID)
{
// Deduplicate: only add words whose start time is after the last stored end time.
// Cartesia sends one timestamps message per audio chunk and may repeat words.
const float LastEnd = Task.WordTimestamps.IsEmpty() ? 0.f : Task.WordTimestamps.Last().End;
for (const FWordTimestamp& WTS : TS)
{
if (WTS.Start >= LastEnd - 0.01f)
Task.WordTimestamps.Add(WTS);
}
break;
}
}
});
}
}
return;
}
if (Type.Equals(TEXT("done"), ESearchCase::IgnoreCase)) if (Type.Equals(TEXT("done"), ESearchCase::IgnoreCase))
{ {
bool bDone = false; bool bDone = false;
@ -353,8 +400,7 @@ void UCartesiaTTSManager::StartStreamingGeneration(int32 TaskID, const FString&
Self->TTSError(FString::Printf(TEXT("[Cartesia][%d] error: %s"), TaskID, *ErrorMsg)); Self->TTSError(FString::Printf(TEXT("[Cartesia][%d] error: %s"), TaskID, *ErrorMsg));
Self->CloseAndRemoveSocket(TaskID, true); Self->CloseAndRemoveSocket(TaskID, true);
return; return;
} } });
});
/*WS->OnRawMessage().AddLambda([WeakThis, TaskID](const void* Data, SIZE_T Size, SIZE_T BytesRemaining) /*WS->OnRawMessage().AddLambda([WeakThis, TaskID](const void* Data, SIZE_T Size, SIZE_T BytesRemaining)
{ {

90
Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Private/RealtimeAPI/RealtimeAPI_TTSManager.cpp

@ -7,4 +7,94 @@ void URealtimeAPI_TTSManager::InitTTSManager(UTTSBaseConfig* InTSSConfig, bool D
{ {
UTTSManagerBase::InitTTSManager(InTSSConfig, DebugMode); UTTSManagerBase::InitTTSManager(InTSSConfig, DebugMode);
SetTTSState(ETTSState::Ready); SetTTSState(ETTSState::Ready);
}
void URealtimeAPI_TTSManager::AddAudioChunk(const TArray<uint8> AudioChunk, bool IsFinal)
{
// Flush any subtitle text that arrived before this first audio chunk created a task.
// Call the base implementation first so the task is created, then update OriginalText.
Super::AddAudioChunk(AudioChunk, IsFinal);
if (!PendingSubtitleText.IsEmpty())
{
FScopeLock Lock(&TaskQueueCriticalSection);
if (ActiveTasks.Num() > 0)
{
ActiveTasks.Last().OriginalText += PendingSubtitleText;
PendingSubtitleText.Empty();
}
}
}
void URealtimeAPI_TTSManager::AddTextChunk(const FString& TextChunk, bool IsFinal)
{
// Do not route through the base class: it would either create a TTS generation task
// (bSupportsStreamedInput=false path) or a competing streaming task (true path),
// both of which leave a phantom task in ActiveTasks that blocks playback forever.
bool bShouldSave = false;
bool bCacheHit = false;
FTTSTask TaskSnapshot;
{
FScopeLock Lock(&TaskQueueCriticalSection);
const FString FullText = PendingSubtitleText + TextChunk;
if (ActiveTasks.Num() > 0)
{
// AI audio path: a task already exists from AddAudioChunk.
// Update its subtitle text; handle cache save/load when text is final.
FTTSTask& Current = ActiveTasks.Last();
PendingSubtitleText.Empty();
Current.OriginalText += FullText;
if (IsFinal)
{
Current.Text = Current.OriginalText;
if (TTSConfig && TTSConfig->GlobalTTSSettings.UseCacheSystem)
{
if (!Current.bIsGenerating)
{
// Audio already done: save to cache now.
TaskSnapshot = Current;
bShouldSave = true;
}
// else: audio still arriving — OnGeneratedAudioChunkReceived saves
// once it sees Text is non-empty on the last audio chunk.
}
}
}
else if (IsFinal && TTSConfig && TTSConfig->GlobalTTSSettings.UseCacheSystem)
{
// Cache path: no AddAudioChunk was called. Create a task and try to load
// cached audio. If the cache misses, the task is discarded (caller should
// not reach here without cached audio available in pure-cache mode).
PendingSubtitleText.Empty();
FTTSTask NewTask;
NewTask.TaskID = NextTaskID++;
NewTask.Text = FullText;
NewTask.OriginalText = FullText;
NewTask.bIsGenerating = true;
if (TryLoadCachedAudioIntoTask(NewTask))
{
NewTask.bIsGenerating = false;
NewTask.AudioStartByteOffset = 0;
ActiveTasks.Add(NewTask);
bReceivedFinalInput = true;
bCacheHit = true;
}
// Cache miss in pure-cache mode: nothing to play, task discarded.
}
else
{
// Audio task not created yet; buffer until AddAudioChunk creates it.
PendingSubtitleText += TextChunk;
}
}
if (bShouldSave)
SaveTaskAudioToCache(TaskSnapshot);
if (bCacheHit)
CheckPlayback();
} }

333
Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Private/TTSManagerBase.cpp

@ -5,8 +5,14 @@
#include "AvatarSoundWave.h" #include "AvatarSoundWave.h"
#include "Misc/App.h" #include "Misc/App.h"
#include "Misc/SecureHash.h" #include "Misc/SecureHash.h"
#include "FL_AvatarCoreShared.h"
#include "AvatarCoreSharedEnums.h"
#include "Kismet/GameplayStatics.h" #include "Kismet/GameplayStatics.h"
#include "HAL/FileManager.h" #include "HAL/FileManager.h"
#include "Dom/JsonObject.h"
#include "Serialization/JsonSerializer.h"
#include "Serialization/JsonWriter.h"
#include "Serialization/JsonReader.h"
UTTSManagerBase::UTTSManagerBase() UTTSManagerBase::UTTSManagerBase()
: RegisteredAudioComponent(nullptr) : RegisteredAudioComponent(nullptr)
@ -294,6 +300,12 @@ void UTTSManagerBase::WipeLastCachedFile()
{ {
IFileManager::Get().Delete(*NewestFullPath, /*RequireExists=*/false, /*EvenReadOnly=*/true, /*Quiet=*/true); IFileManager::Get().Delete(*NewestFullPath, /*RequireExists=*/false, /*EvenReadOnly=*/true, /*Quiet=*/true);
UE_LOG(LogTemp, Warning, TEXT("File deleted: %s"), *NewestFullPath); UE_LOG(LogTemp, Warning, TEXT("File deleted: %s"), *NewestFullPath);
const FString SidecarPath = FPaths::ChangeExtension(NewestFullPath, TEXT("json"));
if (FPaths::FileExists(SidecarPath))
{
IFileManager::Get().Delete(*SidecarPath, /*RequireExists=*/false, /*EvenReadOnly=*/true, /*Quiet=*/true);
}
} }
} }
@ -306,6 +318,9 @@ void UTTSManagerBase::OnTTSManagerFinished()
ProceduralSoundWave->Duration = 0.0f; ProceduralSoundWave->Duration = 0.0f;
} }
AccumulatedSubtitleText.Empty();
StreamingInputBuffer.Empty();
SetTTSState(ETTSState::Ready); SetTTSState(ETTSState::Ready);
} }
@ -365,6 +380,8 @@ void UTTSManagerBase::ClearTTS()
// Reset state variables // Reset state variables
bReceivedFinalInput = false; bReceivedFinalInput = false;
AccumulatedSubtitleText.Empty();
StreamingInputBuffer.Empty();
// Set state to Ready // Set state to Ready
SetTTSState(ETTSState::Ready); SetTTSState(ETTSState::Ready);
@ -378,7 +395,7 @@ void UTTSManagerBase::SetCachingEnabled(bool bEnabled)
TTSConfig->GlobalTTSSettings.UseCacheSystem = bEnabled; TTSConfig->GlobalTTSSettings.UseCacheSystem = bEnabled;
} }
void UTTSManagerBase::SetLanguage(ETTSLanguage NewLanguage) void UTTSManagerBase::SetTTSLanguage(ELanguage NewLanguage)
{ {
if (!TTSConfig) return; if (!TTSConfig) return;
TTSConfig->GlobalTTSSettings.Language = NewLanguage; TTSConfig->GlobalTTSSettings.Language = NewLanguage;
@ -416,6 +433,59 @@ void UTTSManagerBase::AddTextChunk(const FString& TextChunk, bool IsFinal)
if (bShouldStream) if (bShouldStream)
{ {
// Accumulate into buffer and flush only at word boundaries to allow multi-word replacements
StreamingInputBuffer += TextChunk;
FString ToSend;
if (IsFinal)
{
ToSend = StreamingInputBuffer;
StreamingInputBuffer.Empty();
}
else
{
// Find last word boundary scanning from the end
int32 LastBoundary = -1;
for (int32 i = StreamingInputBuffer.Len() - 1; i >= 0; --i)
{
TCHAR C = StreamingInputBuffer[i];
if (C == ' ' || C == '.' || C == ',' || C == '!' || C == '?' || C == ':' || C == ';')
{
LastBoundary = i;
break;
}
}
if (LastBoundary >= 0)
{
ToSend = StreamingInputBuffer.Left(LastBoundary + 1);
StreamingInputBuffer = StreamingInputBuffer.Mid(LastBoundary + 1);
}
}
// Nothing ready yet: still accumulating mid-word
if (ToSend.IsEmpty())
{
if (!IsFinal) return;
if (!bHasActiveStreamingTask) return;
// IsFinal + active task with empty ToSend: fall through to close the task
}
// Apply word replacements to the flushed portion
FString ProcessedChunk = ToSend;
if (TTSConfig && TTSConfig->GlobalTTSSettings.WordReplacements.Num() > 0)
{
for (const FString& Entry : TTSConfig->GlobalTTSSettings.WordReplacements)
{
FString From, To;
if (Entry.Split(TEXT("|"), &From, &To))
{
From = From.TrimStartAndEnd();
To = To.TrimStartAndEnd();
if (!From.IsEmpty())
ProcessedChunk.ReplaceInline(*From, *To, ESearchCase::IgnoreCase);
}
}
}
bool bStartedNewStreamingTask = false; bool bStartedNewStreamingTask = false;
@ -429,8 +499,8 @@ void UTTSManagerBase::AddTextChunk(const FString& TextChunk, bool IsFinal)
{ {
FTTSTask NewTask; FTTSTask NewTask;
NewTask.TaskID = NextTaskID++; NewTask.TaskID = NextTaskID++;
NewTask.OriginalText = TextChunk; NewTask.OriginalText = ToSend;
NewTask.Text = TextChunk; NewTask.Text = ProcessedChunk;
NewTask.bIsStreaming = !IsFinal; NewTask.bIsStreaming = !IsFinal;
NewTask.bIsGenerating = true; NewTask.bIsGenerating = true;
ActiveTasks.Add(NewTask); ActiveTasks.Add(NewTask);
@ -440,12 +510,20 @@ void UTTSManagerBase::AddTextChunk(const FString& TextChunk, bool IsFinal)
} }
else else
{ {
CurrentTask.OriginalText += TextChunk; // CurrentTask is a local copy; update the actual element in ActiveTasks
CurrentTask.Text += TextChunk; for (FTTSTask& ActiveTask : ActiveTasks)
if (IsFinal) {
if (ActiveTask.TaskID == StreamingTaskID)
{ {
CurrentTask.bIsStreaming = false; ActiveTask.OriginalText += ToSend;
ActiveTask.Text += ProcessedChunk;
if (IsFinal)
{
ActiveTask.bIsStreaming = false;
}
break;
} }
}
} }
if (StreamingTaskID != -1) if (StreamingTaskID != -1)
@ -453,11 +531,11 @@ void UTTSManagerBase::AddTextChunk(const FString& TextChunk, bool IsFinal)
if (bStartedNewStreamingTask) if (bStartedNewStreamingTask)
{ {
SetTTSState(ETTSState::Producing); SetTTSState(ETTSState::Producing);
StartStreamingGeneration(StreamingTaskID, TextChunk, IsFinal); StartStreamingGeneration(StreamingTaskID, ProcessedChunk, IsFinal);
} }
else else
{ {
UpdateStreamingText(StreamingTaskID, TextChunk, IsFinal); UpdateStreamingText(StreamingTaskID, ProcessedChunk, IsFinal);
} }
return; return;
} }
@ -547,16 +625,6 @@ void UTTSManagerBase::OnGeneratedAudioChunkReceived(FTTSTask& Task, const TArray
ProcessedAudio = AudioData; ProcessedAudio = AudioData;
} }
// Run through WebRTC APM for chunk-boundary smoothing (NS + HPF)
if (WebRTCChannel && WebRTCChannel->IsInitialized() && ProcessedAudio.Num() > 0)
{
TArray<uint8> WebRTCProcessed = WebRTCChannel->ProcessTTSAudio(ProcessedAudio);
if (WebRTCProcessed.Num() > 0)
{
ProcessedAudio = MoveTemp(WebRTCProcessed);
}
}
// Append audio data to the owning ActiveTask buffer, not directly to the output queue // Append audio data to the owning ActiveTask buffer, not directly to the output queue
FTTSTask* TargetTaskPtr = nullptr; FTTSTask* TargetTaskPtr = nullptr;
{ {
@ -577,6 +645,12 @@ void UTTSManagerBase::OnGeneratedAudioChunkReceived(FTTSTask& Task, const TArray
return; return;
} }
// Record where speech audio starts (before the first real chunk is appended)
if (TargetTaskPtr->AudioStartByteOffset < 0 && ProcessedAudio.Num() > 0)
{
TargetTaskPtr->AudioStartByteOffset = TargetTaskPtr->AudioData.Num();
}
// Append first // Append first
TargetTaskPtr->AudioData.Append(ProcessedAudio); TargetTaskPtr->AudioData.Append(ProcessedAudio);
@ -604,9 +678,16 @@ void UTTSManagerBase::OnGeneratedAudioChunkReceived(FTTSTask& Task, const TArray
TargetTaskPtr->bIsComplete = true; TargetTaskPtr->bIsComplete = true;
} }
} }
if (IsLastChunk && TTSConfig->GlobalTTSSettings.UseCacheSystem && !bIsPreCaching) if (IsLastChunk && TTSConfig->GlobalTTSSettings.UseCacheSystem && !bIsPreCaching && TargetTaskPtr && !TargetTaskPtr->Text.IsEmpty())
{ {
SaveTaskAudioToCache(Task); // Use TargetTaskPtr (actual task) not Task (may be a stale local copy from AddAudioChunk).
// TargetTaskPtr has the complete AudioData including the last chunk just appended above,
// and the correct Text field. Access outside the lock is safe: bIsGenerating=false means
// no further audio appends can occur on this task.
// Note: for RealtimeAPI, Text may still be empty here if the final text chunk has not
// arrived yet — in that case, RealtimeAPI_TTSManager::AddTextChunk triggers the save
// once both streams are finalised.
SaveTaskAudioToCache(*TargetTaskPtr);
} }
UTTSManagerBase::CheckPlayback(); UTTSManagerBase::CheckPlayback();
} }
@ -739,6 +820,22 @@ void UTTSManagerBase::OnAvatarSoundwaveBufferUnderun()
SetTTSState(ETTSState::WaitingForChunks); SetTTSState(ETTSState::WaitingForChunks);
TTSLog(TEXT("Stopping AudioComponent playback")); TTSLog(TEXT("Stopping AudioComponent playback"));
RegisteredAudioComponent->Stop(); RegisteredAudioComponent->Stop();
// Stop() wipes the ProceduralSoundWave internal buffer. Reset QueueCursor to
// PlaybackCursor so those bytes can be re-queued by FlushToAudioQueue.
// Clear QueuedSegments since the audio device will no longer render those bytes.
{
FScopeLock TaskLock(&TaskQueueCriticalSection);
for (FTTSTask& Task : ActiveTasks)
{
Task.QueueCursor = Task.PlaybackCursor;
}
}
{
FScopeLock SegLock(&SegmentQueueCriticalSection);
QueuedSegments.Empty();
}
if (WebRTCChannel) WebRTCChannel->Reset();
} }
} }
@ -909,18 +1006,42 @@ bool UTTSManagerBase::FlushToAudioQueue()
if (ProceduralSoundWave) if (ProceduralSoundWave)
{ {
TTSLog(FString::Printf(TEXT("Queueing %d bytes for task %d"), BytesToQueue, Curr.TaskID)); TTSLog(FString::Printf(TEXT("Queueing %d bytes for task %d"), BytesToQueue, Curr.TaskID));
// Queue the available tail for this task // Apply WebRTC HPF smoothing for live playback; Task.AudioData stays raw for clean caching
ProceduralSoundWave->QueueAudio(Curr.AudioData.GetData() + BytesQueuedSoFar, BytesToQueue); TArray<uint8> AudioToQueue(Curr.AudioData.GetData() + BytesQueuedSoFar, BytesToQueue);
if (WebRTCChannel && WebRTCChannel->IsInitialized())
{
TArray<uint8> Processed = WebRTCChannel->ProcessTTSAudio(AudioToQueue);
if (Processed.Num() > 0)
AudioToQueue = MoveTemp(Processed);
}
ProceduralSoundWave->QueueAudio(AudioToQueue.GetData(), AudioToQueue.Num());
// Update duration accurately: seconds = bytes / (bytesPerSample * channels * sampleRate) // Update duration accurately: seconds = bytes / (bytesPerSample * channels * sampleRate)
const float DurationInc = static_cast<float>(BytesToQueue) / static_cast<float>(BytesPerSecond); const float DurationInc = static_cast<float>(AudioToQueue.Num()) / static_cast<float>(BytesPerSecond);
ProceduralSoundWave->Duration += DurationInc; ProceduralSoundWave->Duration += DurationInc;
// Drain WebRTC accumulator on the final segment so no audio is held back.
// Use !bIsGenerating (not bIsComplete) — bIsComplete requires PlaybackCursor to catch up,
// which hasn't happened yet. bIsGenerating=false means the API sent all chunks.
if (!Curr.bIsGenerating && (Curr.QueueCursor + BytesToQueue >= Curr.AudioData.Num()))
{
if (WebRTCChannel && WebRTCChannel->IsInitialized())
{
TArray<uint8> Tail = WebRTCChannel->Flush();
if (Tail.Num() > 0)
{
ProceduralSoundWave->QueueAudio(Tail.GetData(), Tail.Num());
ProceduralSoundWave->Duration += static_cast<float>(Tail.Num()) / static_cast<float>(BytesPerSecond);
}
}
}
} }
// Advance queue cursor but DO NOT mark as rendered yet // Capture offset before advancing the cursor so ConsumeRenderedBytes can map bytes→time
const int32 SegmentStartOffset = Curr.QueueCursor;
// Advance by AudioData bytes (keeps PlaybackCursor in AudioData coordinate space for subtitle timing)
Curr.QueueCursor += BytesToQueue; Curr.QueueCursor += BytesToQueue;
// Track this queued segment for precise playback accounting // Track this queued segment for precise playback accounting
{ {
FScopeLock SegLock(&SegmentQueueCriticalSection); FScopeLock SegLock(&SegmentQueueCriticalSection);
QueuedSegments.Add({ Curr.TaskID, BytesToQueue }); QueuedSegments.Add({ Curr.TaskID, BytesToQueue, SegmentStartOffset });
} }
bQueuedAny = true; bQueuedAny = true;
@ -1103,8 +1224,7 @@ void UTTSManagerBase::ProcessText(bool IsFinal)
} }
if (IsFinal && !Text.IsEmpty()) if (IsFinal && !Text.IsEmpty())
{ {
FString Sentence = Text; const FString Sentence = Text;
Sentence = ProcessTextForGeneration(Sentence);
if (!Sentence.IsEmpty()) if (!Sentence.IsEmpty())
{ {
// Directly add to PendingTasks queue instead of redundant SentenceQueue // Directly add to PendingTasks queue instead of redundant SentenceQueue
@ -1280,10 +1400,21 @@ FString UTTSManagerBase::GetCacheDirectory() const
FString UTTSManagerBase::GetCacheFilePath(const FString& InText) const FString UTTSManagerBase::GetCacheFilePath(const FString& InText) const
{ {
const FString Hash = ComputeTaskHash(InText); UE_LOG(LogTemp, Warning, TEXT("InText is empty therefore hash cannot be created, using invalidCacheNoText instead."));
FString Hash;
if (InText.IsEmpty())
Hash = "invalidCacheNoText";
else
Hash = ComputeTaskHash(InText);
return FPaths::Combine(GetCacheDirectory(), Hash + TEXT(".wav")); return FPaths::Combine(GetCacheDirectory(), Hash + TEXT(".wav"));
} }
FString UTTSManagerBase::GetCacheSidecarPath(const FString& InText) const
{
const FString Hash = ComputeTaskHash(InText);
return FPaths::Combine(GetCacheDirectory(), Hash + TEXT(".json"));
}
bool UTTSManagerBase::HasCacheFile(const FString& InText) bool UTTSManagerBase::HasCacheFile(const FString& InText)
{ {
const FString FilePath = GetCacheFilePath(ProcessTextForGeneration(InText)); const FString FilePath = GetCacheFilePath(ProcessTextForGeneration(InText));
@ -1380,6 +1511,32 @@ bool UTTSManagerBase::SaveTaskAudioToCache(const FTTSTask& Task)
if (bOk) if (bOk)
{ {
TTSLog(FString::Printf(TEXT("Saved cache: %s (%d bytes)"), *FilePath, Wav.Num())); TTSLog(FString::Printf(TEXT("Saved cache: %s (%d bytes)"), *FilePath, Wav.Num()));
if (Task.WordTimestamps.Num() > 0)
{
TSharedRef<FJsonObject> Root = MakeShared<FJsonObject>();
TArray<TSharedPtr<FJsonValue>> WordsArray;
for (const FWordTimestamp& WT : Task.WordTimestamps)
{
TSharedRef<FJsonObject> Entry = MakeShared<FJsonObject>();
Entry->SetStringField(TEXT("word"), WT.Word);
Entry->SetNumberField(TEXT("start"), WT.Start);
Entry->SetNumberField(TEXT("end"), WT.End);
WordsArray.Add(MakeShared<FJsonValueObject>(Entry));
}
Root->SetArrayField(TEXT("words"), WordsArray);
FString JsonStr;
TSharedRef<TJsonWriter<>> Writer = TJsonWriterFactory<>::Create(&JsonStr);
FJsonSerializer::Serialize(Root, Writer);
const FString SidecarPath = GetCacheSidecarPath(Task.Text);
if (!FFileHelper::SaveStringToFile(JsonStr, *SidecarPath))
{
TTSLog(FString::Printf(TEXT("Failed to save timestamp sidecar: %s"), *SidecarPath));
}
}
} }
else else
{ {
@ -1413,6 +1570,37 @@ bool UTTSManagerBase::TryLoadCachedAudioIntoTask(FTTSTask& Task)
Task.AudioData.Append(DataPtr, DataSize); Task.AudioData.Append(DataPtr, DataSize);
IFileManager::Get().SetTimeStamp(*FilePath, FDateTime::Now()); IFileManager::Get().SetTimeStamp(*FilePath, FDateTime::Now());
const FString SidecarPath = GetCacheSidecarPath(Task.Text);
if (FPaths::FileExists(SidecarPath))
{
FString JsonStr;
if (FFileHelper::LoadFileToString(JsonStr, *SidecarPath))
{
TSharedPtr<FJsonObject> Root;
TSharedRef<TJsonReader<>> Reader = TJsonReaderFactory<>::Create(JsonStr);
if (FJsonSerializer::Deserialize(Reader, Root) && Root.IsValid())
{
const TArray<TSharedPtr<FJsonValue>>* WordsArr;
if (Root->TryGetArrayField(TEXT("words"), WordsArr))
{
for (const TSharedPtr<FJsonValue>& Val : *WordsArr)
{
const TSharedPtr<FJsonObject>* Obj;
if (Val->TryGetObject(Obj))
{
FWordTimestamp WT;
(*Obj)->TryGetStringField(TEXT("word"), WT.Word);
(*Obj)->TryGetNumberField(TEXT("start"), WT.Start);
(*Obj)->TryGetNumberField(TEXT("end"), WT.End);
Task.WordTimestamps.Add(WT);
}
}
}
}
}
}
return true; return true;
} }
@ -1421,6 +1609,15 @@ void UTTSManagerBase::ConsumeRenderedBytes(int32 Bytes)
if (Bytes <= 0) if (Bytes <= 0)
return; return;
if (!TTSConfig)
return;
const int32 BytesPerSample = 2; // 16-bit PCM
const int32 EffSR = (TTSConfig->GlobalTTSSettings.ResampleToSampleRate > 0)
? TTSConfig->GlobalTTSSettings.ResampleToSampleRate
: TTSConfig->GlobalTTSSettings.AudioSampleRate;
const int32 BytesPerSecond = EffSR * TTSConfig->GlobalTTSSettings.AudioNumChannels * BytesPerSample;
// Drain the queued segment FIFO while updating per-task playback // Drain the queued segment FIFO while updating per-task playback
// Lock order: SegmentQueue first, then TaskQueue to avoid deadlock with FlushToAudioQueue // Lock order: SegmentQueue first, then TaskQueue to avoid deadlock with FlushToAudioQueue
FScopeLock SegLock(&SegmentQueueCriticalSection); FScopeLock SegLock(&SegmentQueueCriticalSection);
@ -1450,6 +1647,84 @@ void UTTSManagerBase::ConsumeRenderedBytes(int32 Bytes)
{ {
Task.bIsComplete = true; Task.bIsComplete = true;
} }
// --- Subtitle event ---
TArray<FString> OrigWords;
Task.OriginalText.ParseIntoArray(OrigWords, TEXT(" "), true);
if (OrigWords.Num() > 0)
{
int32 NewWordIdx = Task.LastSubtitleWordIndex;
// TimeNow is relative to the start of Cartesia's audio (not the task buffer start),
// so subtract any silence that was prepended before the first real chunk.
const int32 SpeechStart = (Task.AudioStartByteOffset >= 0) ? Task.AudioStartByteOffset : 0;
const int32 SpeechCursor = FMath::Max(Task.PlaybackCursor - SpeechStart, 0);
const int32 SpeechTotal = FMath::Max(Task.AudioData.Num() - SpeechStart, 0);
if (Task.WordTimestamps.Num() > 0 && BytesPerSecond > 0)
{
// Timestamp path: advance at most one word per callback so timestamps that
// arrived before audio started playing don't all fire in one burst.
const float TimeNow = (float)SpeechCursor / (float)BytesPerSecond;
const int32 NextWi = Task.LastSubtitleWordIndex + 1;
if (NextWi < Task.WordTimestamps.Num() && Task.WordTimestamps[NextWi].Start <= TimeNow)
{
NewWordIdx = NextWi;
}
}
else
{
// Fallback: proportional interpolation, also one word at a time
const float Fraction = (SpeechTotal > 0)
? FMath::Clamp((float)SpeechCursor / (float)SpeechTotal, 0.f, 1.f)
: 0.f;
const int32 ProportionalIdx = FMath::Clamp(
FMath::FloorToInt(Fraction * OrigWords.Num()) - 1,
Task.LastSubtitleWordIndex, OrigWords.Num() - 1);
if (ProportionalIdx > Task.LastSubtitleWordIndex)
NewWordIdx = Task.LastSubtitleWordIndex + 1;
}
if (NewWordIdx > Task.LastSubtitleWordIndex)
{
FString ChunkText;
for (int32 wi = Task.LastSubtitleWordIndex + 1; wi <= NewWordIdx; ++wi)
{
if (OrigWords.IsValidIndex(wi))
{
if (!ChunkText.IsEmpty()) ChunkText += TEXT(" ");
ChunkText += OrigWords[wi];
}
}
Task.LastSubtitleWordIndex = NewWordIdx;
if (!ChunkText.IsEmpty())
{
AccumulatedSubtitleText += (AccumulatedSubtitleText.IsEmpty() ? TEXT("") : TEXT(" ")) + ChunkText;
}
const bool bIsFinal = Task.bIsComplete
&& Task.PlaybackCursor >= Task.AudioData.Num()
&& QueuedSegments.Num() <= 1;
if (!ChunkText.IsEmpty() || bIsFinal)
{
const int32 FireTaskID = Task.TaskID;
FString ChunkCopy = ChunkText;
FString AccumCopy = AccumulatedSubtitleText;
TWeakObjectPtr<UTTSManagerBase> WeakThis(this);
AsyncTask(ENamedThreads::GameThread, [WeakThis, FireTaskID, ChunkCopy, AccumCopy, bIsFinal]()
{
if (WeakThis.IsValid())
{
WeakThis->OnTTSSubtitle.Broadcast(FireTaskID, ChunkCopy, AccumCopy, bIsFinal);
}
});
}
}
}
// --- End subtitle event ---
break; break;
} }
} }

35
Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Private/TTSWebRTCChannel.cpp

@ -53,9 +53,8 @@ void FTTSWebRTCChannel::Configure()
Config.echo_canceller.enabled = false; // AEC handled by AvatarCore_STT Config.echo_canceller.enabled = false; // AEC handled by AvatarCore_STT
Config.pre_amplifier.enabled = false; Config.pre_amplifier.enabled = false;
Config.high_pass_filter.enabled = true; // removes DC offset / low-freq rumble Config.high_pass_filter.enabled = true; // removes DC offset / low-freq rumble
Config.noise_suppression.enabled = true; // smooths inter-chunk artifacts Config.noise_suppression.enabled = false; // causes artifacts on clean TTS audio
Config.noise_suppression.level = webrtc::AudioProcessing::Config::NoiseSuppression::kLow; Config.transient_suppression.enabled = false; // designed for mic capture, not synthesis
Config.transient_suppression.enabled = true; // suppresses click artifacts at chunk boundaries
Config.gain_controller1.enabled = false; Config.gain_controller1.enabled = false;
Config.gain_controller2.enabled = false; Config.gain_controller2.enabled = false;
@ -79,6 +78,36 @@ void FTTSWebRTCChannel::Reset()
} }
} }
// ---------------------------------------------------------------------------
TArray<uint8> FTTSWebRTCChannel::Flush()
{
FScopeLock Lock(&CriticalSection);
if (Accumulator.Num() == 0 || !bInitialized || !AudioProcessing || FrameSamplesTotal <= 0)
return {};
// Zero-pad to a complete frame
while (Accumulator.Num() < FrameSamplesTotal)
Accumulator.Add(0);
webrtc::StreamConfig StreamCfg(WebRTCRate, NumChannels);
TArray<int16> FrameBuf;
FrameBuf.SetNumUninitialized(FrameSamplesTotal);
FMemory::Memcpy(FrameBuf.GetData(), Accumulator.GetData(), FrameSamplesTotal * sizeof(int16));
AudioProcessing->ProcessStream(FrameBuf.GetData(), StreamCfg, StreamCfg, FrameBuf.GetData());
Accumulator.Reset();
TArray<int16> FinalInt16;
if (SampleRate != WebRTCRate)
ResampleInt16Linear(FrameBuf, WebRTCRate, SampleRate, FinalInt16);
else
FinalInt16 = MoveTemp(FrameBuf);
TArray<uint8> Out;
Out.SetNumUninitialized(FinalInt16.Num() * 2);
FMemory::Memcpy(Out.GetData(), FinalInt16.GetData(), Out.Num());
return Out;
}
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
TArray<uint8> FTTSWebRTCChannel::ProcessTTSAudio(const TArray<uint8>& InPCM16) TArray<uint8> FTTSWebRTCChannel::ProcessTTSAudio(const TArray<uint8>& InPCM16)
{ {

11
Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Public/RealtimeAPI/RealtimeAPI_TTSManager.h

@ -17,5 +17,14 @@ class AVATARCORE_TTS_API URealtimeAPI_TTSManager : public UTTSManagerBase
public: public:
// UTTSManagerBase overrides // UTTSManagerBase overrides
virtual void InitTTSManager(UTTSBaseConfig* InTSSConfig, bool DebugMode) override; virtual void InitTTSManager(UTTSBaseConfig* InTSSConfig, bool DebugMode) override;
// In the RealtimeAPI flow, audio comes directly via AddAudioChunk while text is piped
// in separately for subtitle display. These overrides keep text and audio on the same
// task without creating additional TTS generation tasks.
virtual void AddTextChunk(const FString& TextChunk, bool IsFinal) override;
virtual void AddAudioChunk(const TArray<uint8> AudioChunk, bool IsFinal) override;
private:
// Buffer for text that arrives before the first audio chunk creates a task
FString PendingSubtitleText;
}; };

73
Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Public/TTSBaseConfig.h

@ -4,6 +4,7 @@
#include "CoreMinimal.h" #include "CoreMinimal.h"
#include "UObject/NoExportTypes.h" #include "UObject/NoExportTypes.h"
#include "AvatarCoreSharedEnums.h"
#include "TTSBaseConfig.generated.h" #include "TTSBaseConfig.generated.h"
UENUM(BlueprintType) UENUM(BlueprintType)
@ -14,76 +15,6 @@ enum class ECommaSplitRule : uint8 {
FillWord UMETA(DisplayName = "Split on every comma, but add fill words.") FillWord UMETA(DisplayName = "Split on every comma, but add fill words.")
}; };
UENUM(BlueprintType)
enum class ETTSKeepAliveRule : uint8 {
EditorOnly UMETA(DisplayName = "Only keep TTS Module open in Editor Mode"),
Never UMETA(DisplayName = "Never keep TTS Module alive."),
Always UMETA(DisplayName = "Keep TTS Module alive in Editor and Shipping mode.")
};
UENUM(BlueprintType)
enum class ETTSLanguage : uint8
{
NONE UMETA(DisplayName = "Unset"),
en UMETA(DisplayName = "English"),
fr UMETA(DisplayName = "French"),
de UMETA(DisplayName = "German"),
es UMETA(DisplayName = "Spanish"),
pt UMETA(DisplayName = "Portuguese"),
zh UMETA(DisplayName = "Chinese"),
ja UMETA(DisplayName = "Japanese"),
hi UMETA(DisplayName = "Hindi"),
it UMETA(DisplayName = "Italian"),
ko UMETA(DisplayName = "Korean"),
nl UMETA(DisplayName = "Dutch"),
pl UMETA(DisplayName = "Polish"),
ru UMETA(DisplayName = "Russian"),
sv UMETA(DisplayName = "Swedish"),
tr UMETA(DisplayName = "Turkish"),
tl UMETA(DisplayName = "Filipino"),
bg UMETA(DisplayName = "Bulgarian"),
ro UMETA(DisplayName = "Romanian"),
ar UMETA(DisplayName = "Arabic"),
cs UMETA(DisplayName = "Czech"),
el UMETA(DisplayName = "Greek"),
fi UMETA(DisplayName = "Finnish"),
hr UMETA(DisplayName = "Croatian"),
ms UMETA(DisplayName = "Malay"),
sk UMETA(DisplayName = "Slovak"),
da UMETA(DisplayName = "Danish"),
ta UMETA(DisplayName = "Tamil"),
uk UMETA(DisplayName = "Ukrainian"),
hu UMETA(DisplayName = "Hungarian"),
no UMETA(DisplayName = "Norwegian"),
vi UMETA(DisplayName = "Vietnamese"),
bn UMETA(DisplayName = "Bengali"),
th UMETA(DisplayName = "Thai"),
he UMETA(DisplayName = "Hebrew"),
ka UMETA(DisplayName = "Georgian"),
id UMETA(DisplayName = "Indonesian"),
te UMETA(DisplayName = "Telugu"),
gu UMETA(DisplayName = "Gujarati"),
kn UMETA(DisplayName = "Kannada"),
ml UMETA(DisplayName = "Malayalam"),
mr UMETA(DisplayName = "Marathi"),
pa UMETA(DisplayName = "Punjabi"),
};
// Returns the ISO 639-1 code string for the given language enum (e.g. ETTSLanguage::de → "de").
// Derived from the enum value name via UE reflection, so it stays in sync automatically.
static FORCEINLINE FString TTSLanguageToString(ETTSLanguage Language)
{
FString Name = StaticEnum<ETTSLanguage>()->GetNameStringByValue(static_cast<int64>(Language));
// StaticEnum returns the short name (no "ETTSLanguage::" prefix in UE5 GetNameStringByValue)
// but strip the prefix defensively in case the runtime returns the qualified form.
int32 ScopePos;
if (Name.FindLastChar(TEXT(':'), ScopePos))
{
Name = Name.RightChop(ScopePos + 1);
}
return Name.IsEmpty() ? TEXT("en") : Name;
}
USTRUCT(BlueprintType) USTRUCT(BlueprintType)
struct FGlobalTTSSettings struct FGlobalTTSSettings
{ {
@ -134,7 +65,7 @@ struct FGlobalTTSSettings
int32 MaxCharacterForGeneration = 0; int32 MaxCharacterForGeneration = 0;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreTTS|Base", meta = (ExposeOnSpawn = "true")) UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreTTS|Base", meta = (ExposeOnSpawn = "true"))
ETTSLanguage Language = ETTSLanguage::de; ELanguage Language = ELanguage::de;
}; };
class UTTSManagerBase; class UTTSManagerBase;

36
Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Public/TTSManagerBase.h

@ -29,6 +29,15 @@ enum class ETTSState : uint8
FinishPlayback UMETA(DisplayName = "Finish Playback") FinishPlayback UMETA(DisplayName = "Finish Playback")
}; };
USTRUCT()
struct FWordTimestamp
{
GENERATED_BODY()
FString Word;
float Start = 0.f;
float End = 0.f;
};
// TTS Task struct for concurrent generation // TTS Task struct for concurrent generation
USTRUCT(BlueprintType) USTRUCT(BlueprintType)
struct FTTSTask struct FTTSTask
@ -48,6 +57,9 @@ struct FTTSTask
// Queued cursor: how many bytes have been queued to the procedural wave (may be ahead of PlaybackCursor) // Queued cursor: how many bytes have been queued to the procedural wave (may be ahead of PlaybackCursor)
int32 QueueCursor = 0; int32 QueueCursor = 0;
bool bIsComplete = false; // True when all chunks are received and played bool bIsComplete = false; // True when all chunks are received and played
TArray<FWordTimestamp> WordTimestamps; // populated async by Cartesia timestamp messages
int32 LastSubtitleWordIndex = -1; // last word index fired in TTSSubtitle event
int32 AudioStartByteOffset = -1; // byte offset in AudioData where first real audio chunk begins (after any prepended silence)
FTTSTask() : Text(), OriginalText(), TaskID(0) {} FTTSTask() : Text(), OriginalText(), TaskID(0) {}
FTTSTask(const FString& InText, FString& InOriginalText, int32 InTaskID) : Text(InText), OriginalText(InOriginalText), TaskID(InTaskID) {} FTTSTask(const FString& InText, FString& InOriginalText, int32 InTaskID) : Text(InText), OriginalText(InOriginalText), TaskID(InTaskID) {}
}; };
@ -58,6 +70,7 @@ DECLARE_DYNAMIC_MULTICAST_DELEGATE(FOnTTSCacheFinished);
DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FOnTTSError, const FString&, ErrorMessage); DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FOnTTSError, const FString&, ErrorMessage);
DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FOnTTSStateChanged, ETTSState, NewState); DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FOnTTSStateChanged, ETTSState, NewState);
DECLARE_DYNAMIC_MULTICAST_DELEGATE_ThreeParams(FOnTTSAudioChunkForBP, const TArray<float>&, PCMData, int32, SampleRate, int32, NumChannels); DECLARE_DYNAMIC_MULTICAST_DELEGATE_ThreeParams(FOnTTSAudioChunkForBP, const TArray<float>&, PCMData, int32, SampleRate, int32, NumChannels);
DECLARE_DYNAMIC_MULTICAST_DELEGATE_FourParams(FOnTTSSubtitle, int32, TaskID, const FString&, CurrentChunkText, const FString&, AccumulatedSubtitle, bool, bIsFinal);
UCLASS(Abstract, Blueprintable, BlueprintType) UCLASS(Abstract, Blueprintable, BlueprintType)
class AVATARCORE_TTS_API UTTSManagerBase : public UObject class AVATARCORE_TTS_API UTTSManagerBase : public UObject
@ -92,6 +105,10 @@ public:
UPROPERTY(BlueprintAssignable, Category = "AvatarCoreTTS|Events") UPROPERTY(BlueprintAssignable, Category = "AvatarCoreTTS|Events")
FOnTTSCacheUpdate TTSCacheUpdate; FOnTTSCacheUpdate TTSCacheUpdate;
// Fired as words are spoken; CurrentChunkText = words spoken since last event, AccumulatedSubtitle = all words spoken so far
UPROPERTY(BlueprintAssignable, Category = "AvatarCoreTTS|Events")
FOnTTSSubtitle OnTTSSubtitle;
// Initializes the Manager and set Config file // Initializes the Manager and set Config file
UFUNCTION(BlueprintCallable, Category = "AvatarCoreTTS|Lifecycle") UFUNCTION(BlueprintCallable, Category = "AvatarCoreTTS|Lifecycle")
virtual void InitTTSManager(UTTSBaseConfig* InTSSConfig, bool DebugMode); virtual void InitTTSManager(UTTSBaseConfig* InTSSConfig, bool DebugMode);
@ -109,7 +126,7 @@ public:
// Set the language used for TTS generation // Set the language used for TTS generation
UFUNCTION(BlueprintCallable, Category = "AvatarCoreTTS|Settings") UFUNCTION(BlueprintCallable, Category = "AvatarCoreTTS|Settings")
void SetLanguage(ETTSLanguage NewLanguage); void SetTTSLanguage(ELanguage NewLanguage);
// Is the TTSManager currently busy // Is the TTSManager currently busy
UFUNCTION(BlueprintCallable, Category = "AvatarCoreTTS|Helper") UFUNCTION(BlueprintCallable, Category = "AvatarCoreTTS|Helper")
@ -217,8 +234,6 @@ protected:
void SetTTSState(ETTSState NewState); void SetTTSState(ETTSState NewState);
void AddEndSilence(FTTSTask& Task);
// Sentence segmentation // Sentence segmentation
FString PendingText; FString PendingText;
FCriticalSection PendingTextCriticalSection; FCriticalSection PendingTextCriticalSection;
@ -244,9 +259,6 @@ protected:
virtual void AddToRootManager(); virtual void AddToRootManager();
virtual void RemoveFromRootManager(); virtual void RemoveFromRootManager();
// Centralized Audio Queue Management
void AddSilenceToTask(FTTSTask& Task, float SilenceLength);
bool bIsRooted; bool bIsRooted;
// Timers // Timers
@ -281,6 +293,9 @@ private:
TArray<FString> PreCacheSentences; TArray<FString> PreCacheSentences;
// Accumulates streaming text until a word boundary before applying replacements and forwarding
FString StreamingInputBuffer;
TUniquePtr<FTTSWebRTCChannel> WebRTCChannel; TUniquePtr<FTTSWebRTCChannel> WebRTCChannel;
// Timer for polling Ready transition after playback finished // Timer for polling Ready transition after playback finished
@ -294,17 +309,20 @@ protected:
FString GetCacheDirectory() const; FString GetCacheDirectory() const;
// Full cache file path for given text (Hash.wav) // Full cache file path for given text (Hash.wav)
FString GetCacheFilePath(const FString& InText) const; FString GetCacheFilePath(const FString& InText) const;
// Sidecar JSON path for word timestamps (Hash.json, same stem as wav)
FString GetCacheSidecarPath(const FString& InText) const;
// Try to load cached WAV into Task.AudioData (returns true on success) // Try to load cached WAV into Task.AudioData (returns true on success)
bool TryLoadCachedAudioIntoTask(FTTSTask& Task); bool TryLoadCachedAudioIntoTask(FTTSTask& Task);
// Save Task.AudioData as WAV to cache (returns true on success) // Save Task.AudioData as WAV to cache (returns true on success)
bool SaveTaskAudioToCache(const FTTSTask& Task); bool SaveTaskAudioToCache(const FTTSTask& Task);
bool bInitialSilenceDone = false;
void OnGeneratedAudioChunkReceived(FTTSTask& Task, const TArray<uint8>& AudioData, bool IsLastChunk); void OnGeneratedAudioChunkReceived(FTTSTask& Task, const TArray<uint8>& AudioData, bool IsLastChunk);
// Accumulated subtitle text across the current speech session; cleared by ClearTTS
FString AccumulatedSubtitleText;
// Accurate playback accounting without relying on underflow: FIFO of queued segments // Accurate playback accounting without relying on underflow: FIFO of queued segments
struct FQueuedSegment { int32 TaskID; int32 BytesRemaining; }; struct FQueuedSegment { int32 TaskID; int32 BytesRemaining; int32 StartByteOffset; };
FCriticalSection SegmentQueueCriticalSection; FCriticalSection SegmentQueueCriticalSection;
TArray<FQueuedSegment> QueuedSegments; // treated as a queue (pop from front) TArray<FQueuedSegment> QueuedSegments; // treated as a queue (pop from front)

3
Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Public/TTSWebRTCChannel.h

@ -41,6 +41,9 @@ public:
/** Flush and discard any accumulated partial frame. */ /** Flush and discard any accumulated partial frame. */
void Reset(); void Reset();
/** Zero-pad accumulator to a full frame, process it, return the result, then clear the accumulator. Returns empty if accumulator is empty. */
TArray<uint8> Flush();
bool IsInitialized() const { return bInitialized; } bool IsInitialized() const { return bInitialized; }
private: private:

2
Unreal/Plugins/BLogger/Source/BLogger/Public/SLogWidgetState.h

@ -15,7 +15,7 @@ private:
bool IsScrollToBottomEnabled = true; bool IsScrollToBottomEnabled = true;
bool IsAutoRefreshEnabled = true; bool IsAutoRefreshEnabled = true;
TArray<FString> FilteredCategories; TArray<FString> FilteredCategories;
int32 VerbosityFilterMask = 127; int32 VerbosityFilterMask = 126; // VeryVerbose (1) excluded by default
FString SearchText; FString SearchText;
public: public:

BIN
Unreal/Plugins/BTools/Content/Components/CheckProcessComponent.uasset (Stored with Git LFS)

Binary file not shown.

196
Unreal/Plugins/BTools/Source/BTools/Private/BToolsBPLibrary.cpp

@ -15,24 +15,24 @@
#include "BToolsBPLibrary.h" #include "BToolsBPLibrary.h"
#if PLATFORM_WINDOWS #if PLATFORM_WINDOWS
#include <ws2tcpip.h> #include <ws2tcpip.h>
#pragma comment(lib, "ws2_32.lib") #pragma comment(lib, "ws2_32.lib")
#include <iphlpapi.h> #include <iphlpapi.h>
#include "Misc/Paths.h" #include "Misc/Paths.h"
#include <icmpapi.h> #include <icmpapi.h>
#pragma comment(lib, "iphlpapi.lib") #pragma comment(lib, "iphlpapi.lib")
#include <winsock2.h> #include <winsock2.h>
#include "GenericPlatform/GenericWindow.h" #include "GenericPlatform/GenericWindow.h"
#include <TlHelp32.h> #include <TlHelp32.h>
//Create Shortscuts on Runtime //Create Shortscuts on Runtime
#include "Windows/AllowWindowsPlatformTypes.h" #include "Windows/AllowWindowsPlatformTypes.h"
#include "Windows/WindowsHWrapper.h" #include "Windows/WindowsHWrapper.h"
#include <shobjidl.h> #include <shobjidl.h>
#include <shlguid.h> #include <shlguid.h>
#include "Windows/HideWindowsPlatformTypes.h" #include "Windows/HideWindowsPlatformTypes.h"
//For getting Desktop Path //For getting Desktop Path
#include <knownfolders.h> #include <knownfolders.h>
#include <shlobj.h> #include <shlobj.h>
#endif #endif
#include "Components/NamedSlot.h" #include "Components/NamedSlot.h"
#include "Runtime/UMG/public/Blueprint/UserWidget.h" #include "Runtime/UMG/public/Blueprint/UserWidget.h"
@ -119,8 +119,12 @@ bool UBToolsBPLibrary::KillProcessByName(FString Name)
#endif #endif
} }
int UBToolsBPLibrary::KillProcessesByPath(FString Dir) int UBToolsBPLibrary::KillProcessesByPath(FString Dir, bool UseAbsolutePath)
{ {
if (Dir.IsEmpty())
return 0;
#if PLATFORM_WINDOWS #if PLATFORM_WINDOWS
int32 ReturnCode = 0; FString Out, Err; int32 ReturnCode = 0; FString Out, Err;
int32 Killed = 0; int32 Killed = 0;
@ -139,28 +143,37 @@ int UBToolsBPLibrary::KillProcessesByPath(FString Dir)
PROCESSENTRY32 Entry; Entry.dwSize = sizeof(PROCESSENTRY32); PROCESSENTRY32 Entry; Entry.dwSize = sizeof(PROCESSENTRY32);
if (::Process32First(SnapShot, &Entry)) if (::Process32First(SnapShot, &Entry))
{ {
do do
{ {
if (FCString::Stricmp(TEXT("python.exe"), Entry.szExeFile) == 0) HANDLE ProcessHandle = ::OpenProcess(PROCESS_TERMINATE | PROCESS_QUERY_LIMITED_INFORMATION, /*bInherit*/ false, Entry.th32ProcessID);
if (ProcessHandle)
{ {
HANDLE ProcessHandle = ::OpenProcess(PROCESS_TERMINATE | PROCESS_QUERY_LIMITED_INFORMATION, /*bInherit*/ false, Entry.th32ProcessID); WCHAR Buffer[1024]; DWORD Size = UE_ARRAY_COUNT(Buffer);
if (ProcessHandle) if (::QueryFullProcessImageNameW(ProcessHandle, 0, Buffer, &Size))
{ {
WCHAR Buffer[1024]; DWORD Size = UE_ARRAY_COUNT(Buffer); FString ProcPath(Buffer);
if (::QueryFullProcessImageNameW(ProcessHandle, 0, Buffer, &Size)) ProcPath.ReplaceInline(TEXT("/"), TEXT("\\"));
// Case-insensitive prefix check
if (UseAbsolutePath)
{ {
FString ProcPath(Buffer);
ProcPath.ReplaceInline(TEXT("/"), TEXT("\\"));
// Case-insensitive prefix check
if (ProcPath.Len() >= ScriptsDirFull.Len() && ProcPath.Left(ScriptsDirFull.Len()).Compare(ScriptsDirFull, ESearchCase::IgnoreCase) == 0) if (ProcPath.Len() >= ScriptsDirFull.Len() && ProcPath.Left(ScriptsDirFull.Len()).Compare(ScriptsDirFull, ESearchCase::IgnoreCase) == 0)
{ {
::TerminateProcess(ProcessHandle, 0); ::TerminateProcess(ProcessHandle, 0);
++Killed; ++Killed;
} }
} }
::CloseHandle(ProcessHandle); else //Only contains string
{
if (ProcPath.Contains(Dir, ESearchCase::Type::IgnoreCase, ESearchDir::FromStart))
{
::TerminateProcess(ProcessHandle, 0);
++Killed;
}
}
} }
::CloseHandle(ProcessHandle);
} }
} while (::Process32Next(SnapShot, &Entry)); } while (::Process32Next(SnapShot, &Entry));
} }
@ -810,9 +823,9 @@ void UBToolsBPLibrary::IcmpPingAsync(const FString& DomainName, FOnPingComplete
{ {
UE_LOG(LogTemp, Error, TEXT("Failed to resolve domain name: %s"), *DomainName); UE_LOG(LogTemp, Error, TEXT("Failed to resolve domain name: %s"), *DomainName);
AsyncTask(ENamedThreads::GameThread, [CompletionCallback]() AsyncTask(ENamedThreads::GameThread, [CompletionCallback]()
{ {
CompletionCallback.ExecuteIfBound(false, 0.0f); CompletionCallback.ExecuteIfBound(false, 0.0f);
}); });
return; return;
} }
@ -827,9 +840,9 @@ void UBToolsBPLibrary::IcmpPingAsync(const FString& DomainName, FOnPingComplete
{ {
UE_LOG(LogTemp, Error, TEXT("Unable to open ICMP handle")); UE_LOG(LogTemp, Error, TEXT("Unable to open ICMP handle"));
AsyncTask(ENamedThreads::GameThread, [CompletionCallback]() AsyncTask(ENamedThreads::GameThread, [CompletionCallback]()
{ {
CompletionCallback.ExecuteIfBound(false, 0.0f); CompletionCallback.ExecuteIfBound(false, 0.0f);
}); return; }); return;
} }
char SendData[] = "PingData"; char SendData[] = "PingData";
@ -840,9 +853,9 @@ void UBToolsBPLibrary::IcmpPingAsync(const FString& DomainName, FOnPingComplete
UE_LOG(LogTemp, Error, TEXT("Unable to allocate memory for reply buffer")); UE_LOG(LogTemp, Error, TEXT("Unable to allocate memory for reply buffer"));
IcmpCloseHandle(hIcmpFile); IcmpCloseHandle(hIcmpFile);
AsyncTask(ENamedThreads::GameThread, [CompletionCallback]() AsyncTask(ENamedThreads::GameThread, [CompletionCallback]()
{ {
CompletionCallback.ExecuteIfBound(false, 0.0f); CompletionCallback.ExecuteIfBound(false, 0.0f);
}); return; }); return;
} }
// Send the ICMP echo request // Send the ICMP echo request
@ -854,24 +867,24 @@ void UBToolsBPLibrary::IcmpPingAsync(const FString& DomainName, FOnPingComplete
//UE_LOG(LogTemp, Log, TEXT("Sent ICMP message to %s"), *DomainName); //UE_LOG(LogTemp, Log, TEXT("Sent ICMP message to %s"), *DomainName);
//UE_LOG(LogTemp, Log, TEXT("Received %ld bytes from %s in %ld ms"), pEchoReply->DataSize, *DomainName, PingTime); //UE_LOG(LogTemp, Log, TEXT("Received %ld bytes from %s in %ld ms"), pEchoReply->DataSize, *DomainName, PingTime);
AsyncTask(ENamedThreads::GameThread, [CompletionCallback, PingTime]() AsyncTask(ENamedThreads::GameThread, [CompletionCallback, PingTime]()
{ {
CompletionCallback.ExecuteIfBound(true, PingTime); CompletionCallback.ExecuteIfBound(true, PingTime);
}); });
} }
else else
{ {
UE_LOG(LogTemp, Error, TEXT("Ping to %s failed"), *DomainName); UE_LOG(LogTemp, Error, TEXT("Ping to %s failed"), *DomainName);
AsyncTask(ENamedThreads::GameThread, [CompletionCallback]() AsyncTask(ENamedThreads::GameThread, [CompletionCallback]()
{ {
CompletionCallback.ExecuteIfBound(false, 0.0f); CompletionCallback.ExecuteIfBound(false, 0.0f);
}); });
} }
// Cleanup // Cleanup
free(ReplyBuffer); free(ReplyBuffer);
IcmpCloseHandle(hIcmpFile); IcmpCloseHandle(hIcmpFile);
}); });
#endif #endif
} }
@ -1069,36 +1082,39 @@ BOOL CALLBACK MonitorRectProc(HMONITOR hMonitor, HDC hdcMonitor, LPRECT lprcMoni
void UBToolsBPLibrary::BringAppToForeground() void UBToolsBPLibrary::BringAppToForeground()
{ {
#if PLATFORM_WINDOWS #if PLATFORM_WINDOWS
// Try to get the main Unreal window. If not available, fallback to first top-level window. // Try to get the main Unreal window. If not available, fallback to first top-level window.
TSharedPtr<SWindow> MainWindow = FSlateApplication::IsInitialized() ? FSlateApplication::Get().GetActiveTopLevelWindow() : nullptr; TSharedPtr<SWindow> MainWindow = FSlateApplication::IsInitialized() ? FSlateApplication::Get().GetActiveTopLevelWindow() : nullptr;
if (!MainWindow.IsValid()) { if (!MainWindow.IsValid()) {
const TArray<TSharedRef<SWindow>>& TopWindows = FSlateApplication::Get().GetTopLevelWindows(); const TArray<TSharedRef<SWindow>>& TopWindows = FSlateApplication::Get().GetTopLevelWindows();
if (TopWindows.Num() > 0) { if (TopWindows.Num() > 0) {
MainWindow = TopWindows[0]; MainWindow = TopWindows[0];
} }
} }
if (MainWindow.IsValid()) { if (MainWindow.IsValid()) {
TSharedPtr<FGenericWindow> NativeWindow = MainWindow->GetNativeWindow(); TSharedPtr<FGenericWindow> NativeWindow = MainWindow->GetNativeWindow();
if (NativeWindow.IsValid()) { if (NativeWindow.IsValid()) {
HWND hWnd = static_cast<HWND>(NativeWindow->GetOSWindowHandle()); HWND hWnd = static_cast<HWND>(NativeWindow->GetOSWindowHandle());
if (hWnd) { if (hWnd) {
// Attach input to bypass focus restrictions // Attach input to bypass focus restrictions
DWORD CurrentThreadId = GetCurrentThreadId(); DWORD CurrentThreadId = GetCurrentThreadId();
DWORD ForegroundThreadId = GetWindowThreadProcessId(GetForegroundWindow(), nullptr); DWORD ForegroundThreadId = GetWindowThreadProcessId(GetForegroundWindow(), nullptr);
AttachThreadInput(ForegroundThreadId, CurrentThreadId, true); AttachThreadInput(ForegroundThreadId, CurrentThreadId, true);
SetForegroundWindow(hWnd); SetForegroundWindow(hWnd);
SetFocus(hWnd); SetFocus(hWnd);
SetActiveWindow(hWnd); SetActiveWindow(hWnd);
AttachThreadInput(ForegroundThreadId, CurrentThreadId, false); AttachThreadInput(ForegroundThreadId, CurrentThreadId, false);
} else { }
UE_LOG(LogTemp, Warning, TEXT("BringAppToForeground: hWnd is null")); else {
} UE_LOG(LogTemp, Warning, TEXT("BringAppToForeground: hWnd is null"));
} else { }
UE_LOG(LogTemp, Warning, TEXT("BringAppToForeground: NativeWindow invalid")); }
} else {
} else { UE_LOG(LogTemp, Warning, TEXT("BringAppToForeground: NativeWindow invalid"));
UE_LOG(LogTemp, Warning, TEXT("BringAppToForeground: MainWindow invalid")); }
} }
else {
UE_LOG(LogTemp, Warning, TEXT("BringAppToForeground: MainWindow invalid"));
}
#endif #endif
} }
@ -1305,7 +1321,7 @@ void UBToolsBPLibrary::Minimize()
void UBToolsBPLibrary::Maximize() void UBToolsBPLibrary::Maximize()
{ {
//#if !WITH_EDITOR //#if !WITH_EDITOR
UGameEngine* gameEngine = Cast<UGameEngine>(GEngine); UGameEngine* gameEngine = Cast<UGameEngine>(GEngine);
@ -1320,7 +1336,7 @@ void UBToolsBPLibrary::Maximize()
} }
} }
//#endif // !WITH_EDITOR //#endif // !WITH_EDITOR
} }
bool UBToolsBPLibrary::IsFullscreen() bool UBToolsBPLibrary::IsFullscreen()
@ -1372,7 +1388,7 @@ bool UBToolsBPLibrary::IsWindowMaximized()
void UBToolsBPLibrary::MoveWindowToMonitor(int monitorID) void UBToolsBPLibrary::MoveWindowToMonitor(int monitorID)
{ {
//#if !WITH_EDITOR //#if !WITH_EDITOR
if (GEngine && GEngine->GameViewport && GEngine->GameViewport->GetWindow().IsValid()) if (GEngine && GEngine->GameViewport && GEngine->GameViewport->GetWindow().IsValid())
{ {
@ -1402,7 +1418,7 @@ void UBToolsBPLibrary::MoveWindowToMonitor(int monitorID)
} }
} }
} }
//#endif // !WITH_EDITOR //#endif // !WITH_EDITOR
} }
FVector2D UBToolsBPLibrary::GetWidgetScreenPosition(const UUserWidget* Widget) FVector2D UBToolsBPLibrary::GetWidgetScreenPosition(const UUserWidget* Widget)
@ -1417,7 +1433,7 @@ FVector2D UBToolsBPLibrary::GetWidgetScreenPosition(const UUserWidget* Widget)
FGeometry Geometry = Widget->GetCachedGeometry(); FGeometry Geometry = Widget->GetCachedGeometry();
FVector2D LocalCenter = Geometry.GetLocalSize() * 0.5f; FVector2D LocalCenter = Geometry.GetLocalSize() * 0.5f;
FVector2D AbsoluteCenter = Geometry.LocalToAbsolute(LocalCenter); FVector2D AbsoluteCenter = Geometry.LocalToAbsolute(LocalCenter);
return AbsoluteCenter; return AbsoluteCenter;
} }
@ -1425,10 +1441,10 @@ FWidgetLocationResult UBToolsBPLibrary::GetWorldLocationFromUIElement(const UUse
{ {
FWidgetLocationResult Result; FWidgetLocationResult Result;
if (!Widget) return Result; if (!Widget) return Result;
// Hole Widget Position auf Screen // Hole Widget Position auf Screen
FVector2D WidgetScreenPosition = GetWidgetScreenPosition(Widget); FVector2D WidgetScreenPosition = GetWidgetScreenPosition(Widget);
UWorld* World = Widget->GetWorld(); UWorld* World = Widget->GetWorld();
if (!World) return Result; if (!World) return Result;
// Deproject von Screen zu World // Deproject von Screen zu World
@ -1437,7 +1453,7 @@ FWidgetLocationResult UBToolsBPLibrary::GetWorldLocationFromUIElement(const UUse
UE_LOG(LogTemp, Warning, TEXT("WidgetScreenPosition BEFORE adjustments: %f, %f"), WidgetScreenPosition.X, WidgetScreenPosition.Y); UE_LOG(LogTemp, Warning, TEXT("WidgetScreenPosition BEFORE adjustments: %f, %f"), WidgetScreenPosition.X, WidgetScreenPosition.Y);
UE_LOG(LogTemp, Warning, TEXT("Widget Center: %f, %f"), WidgetScreenPosition.X, WidgetScreenPosition.Y); UE_LOG(LogTemp, Warning, TEXT("Widget Center: %f, %f"), WidgetScreenPosition.X, WidgetScreenPosition.Y);
// Update Postions so it is in center of the actual Widget // Update Postions so it is in center of the actual Widget
//WidgetScreenPosition.X = WidgetScreenPosition.X - (Size.X/2);//Widget->GetRenderTransform().Scale.X; //WidgetScreenPosition.X = WidgetScreenPosition.X - (Size.X/2);//Widget->GetRenderTransform().Scale.X;
//WidgetScreenPosition.Y = WidgetScreenPosition.Y - (Size.Y/2); //- Widget->GetRenderTransform().Scale.Y; //WidgetScreenPosition.Y = WidgetScreenPosition.Y - (Size.Y/2); //- Widget->GetRenderTransform().Scale.Y;
@ -1445,7 +1461,7 @@ FWidgetLocationResult UBToolsBPLibrary::GetWorldLocationFromUIElement(const UUse
UE_LOG(LogTemp, Warning, TEXT("WidgetScreenPosition AFTER adjustments: %f, %f"), WidgetScreenPosition.X, WidgetScreenPosition.Y); UE_LOG(LogTemp, Warning, TEXT("WidgetScreenPosition AFTER adjustments: %f, %f"), WidgetScreenPosition.X, WidgetScreenPosition.Y);
//UE_LOG(LogTemp, Warning, TEXT("Widget Absolute Size: %f, %f"), Widget->GetCachedGeometry().GetAbsoluteSize().X, Widget->GetCachedGeometry().GetAbsoluteSize().Y); //UE_LOG(LogTemp, Warning, TEXT("Widget Absolute Size: %f, %f"), Widget->GetCachedGeometry().GetAbsoluteSize().X, Widget->GetCachedGeometry().GetAbsoluteSize().Y);
UE_LOG(LogTemp, Warning, TEXT("Widget Absolute Size: %f, %f"), Size.X, Size.Y); UE_LOG(LogTemp, Warning, TEXT("Widget Absolute Size: %f, %f"), Size.X, Size.Y);
FVector WorldLocation, WorldDirection; FVector WorldLocation, WorldDirection;
PC->DeprojectScreenPositionToWorld( PC->DeprojectScreenPositionToWorld(
WidgetScreenPosition.X, WidgetScreenPosition.X,
@ -1463,16 +1479,16 @@ FVector UBToolsBPLibrary::GetWorldLocationFromUIElementWithRay(const UUserWidget
{ {
FVector Target = FVector::ZeroVector; FVector Target = FVector::ZeroVector;
if (!Widget) return Target; if (!Widget) return Target;
// Get widget screen position // Get widget screen position
FVector2D WidgetScreenPosition = GetWidgetScreenPosition(Widget); FVector2D WidgetScreenPosition = GetWidgetScreenPosition(Widget);
UWorld* World = Widget->GetWorld(); UWorld* World = Widget->GetWorld();
if (!World) return Target; if (!World) return Target;
APlayerController* PC = World->GetFirstPlayerController(); APlayerController* PC = World->GetFirstPlayerController();
if (!PC) return Target; if (!PC) return Target;
// Deproject from screen to world // Deproject from screen to world
FVector WorldLocation, WorldDirection; FVector WorldLocation, WorldDirection;
PC->DeprojectScreenPositionToWorld( PC->DeprojectScreenPositionToWorld(
@ -1481,15 +1497,15 @@ FVector UBToolsBPLibrary::GetWorldLocationFromUIElementWithRay(const UUserWidget
WorldLocation, WorldLocation,
WorldDirection WorldDirection
); );
// Now raycast forward from camera // Now raycast forward from camera
FVector TraceStart = WorldLocation; FVector TraceStart = WorldLocation;
FVector TraceEnd = WorldLocation + (WorldDirection * TraceDistance); FVector TraceEnd = WorldLocation + (WorldDirection * TraceDistance);
FHitResult HitResult; FHitResult HitResult;
FCollisionQueryParams CollisionParams; FCollisionQueryParams CollisionParams;
CollisionParams.AddIgnoredActor(PC->GetPawn()); CollisionParams.AddIgnoredActor(PC->GetPawn());
// Line trace to find world position // Line trace to find world position
if (World->LineTraceSingleByChannel( if (World->LineTraceSingleByChannel(
HitResult, HitResult,
@ -1502,7 +1518,7 @@ FVector UBToolsBPLibrary::GetWorldLocationFromUIElementWithRay(const UUserWidget
// Hit something - return hit location // Hit something - return hit location
return HitResult.Location; return HitResult.Location;
} }
// No hit - return point at fixed distance // No hit - return point at fixed distance
return TraceStart + (WorldDirection * TraceDistance); return TraceStart + (WorldDirection * TraceDistance);
} }

3
Unreal/Plugins/BTools/Source/BTools/Public/BToolsBPLibrary.h

@ -140,8 +140,9 @@ public:
UFUNCTION(BlueprintCallable, Category = "BTools") UFUNCTION(BlueprintCallable, Category = "BTools")
static bool KillProcessByName(FString Name); static bool KillProcessByName(FString Name);
//Kill a process based on its path name. If UseAbolutePath is false all processes containing that FString dir will be killed.
UFUNCTION(BlueprintCallable, Category = "BTools") UFUNCTION(BlueprintCallable, Category = "BTools")
static int KillProcessesByPath(FString Dir); static int KillProcessesByPath(FString Dir, bool UseAbsolutePath = true);
UFUNCTION(BlueprintCallable, Category = "BTools") UFUNCTION(BlueprintCallable, Category = "BTools")
static bool KillProcessesByID(int ProcessID); static bool KillProcessesByID(int ProcessID);

4
Unreal/SPIE_Avatar.uproject

@ -65,6 +65,10 @@
{ {
"Name": "RigLogic", "Name": "RigLogic",
"Enabled": true "Enabled": true
},
{
"Name": "RawInput",
"Enabled": true
} }
], ],
"TargetPlatforms": [ "TargetPlatforms": [

2
Unreal/SyncAvatarCore.bat

@ -49,7 +49,7 @@ if not exist "%SRC_ROOT%\" (
rem List of folders to mirror rem List of folders to mirror
set "FOLDERS=Plugins\AvatarCore_AI Plugins\AvatarCore_Manager Plugins\AvatarCore_MetaHuman Plugins\AvatarCore_STT Plugins\AvatarCore_TTS Plugins\BLogger Plugins\BSettings Plugins\BTools Plugins\RuntimeMetaHumanLipSync_5.6 Content\Project" set "FOLDERS=Plugins\AvatarCore_AI Plugins\AvatarCore_Shared Plugins\AvatarCore_Manager Plugins\AvatarCore_MetaHuman Plugins\AvatarCore_STT Plugins\AvatarCore_TTS Plugins\BLogger Plugins\BSettings Plugins\BTools Plugins\RuntimeMetaHumanLipSync_5.6 Content\Project"
set "ANY_FAIL=0" set "ANY_FAIL=0"

109
Unreal/SyncBackToAvatarCore.bat

@ -0,0 +1,109 @@
@echo off
setlocal EnableExtensions EnableDelayedExpansion
rem ============================================================
rem Mirror specific folders from A (AvatarBase) to B (this repo)
rem - Works relative to this .bat location
rem - Wipes extras in destination (/MIR)
rem - Logs removed extras (shown as *EXTRA File / *EXTRA Dir)
rem - Pauses before exit so console stays open
rem ============================================================
rem Destination root is the folder where this script lives
set "SRC_ROOT=%~dp0"
if "%SRC_ROOT:~-1%"=="\" set "SRC_ROOT=%SRC_ROOT:~0,-1%"
rem Source root relative to this script
set "DST_ROOT=%SRC_ROOT%\..\..\AvatarBase\Unreal"
rem Canonicalize paths
for %%I in ("%SRC_ROOT%") do set "SRC_ROOT=%%~fI"
for %%I in ("%DST_ROOT%") do set "DST_ROOT=%%~fI"
echo ============================================================
echo Robocopy Sync Script
echo ============================================================
echo Source: %SRC_ROOT%
echo Destination: %DST_ROOT%
echo ============================================================
echo.
rem Guard: only intended for subprojects
if /I "%SRC_ROOT%"=="%DST_ROOT%" (
echo ERROR: Source and destination are identical:
echo %SRC_ROOT%
echo This sync script is intended to be run from subprojects only.
echo.
pause
exit /b 1
)
if not exist "%SRC_ROOT%\" (
echo ERROR: Source root does not exist:
echo %SRC_ROOT%
echo.
pause
exit /b 2
)
echo ============================================================
echo Push back data from a project to avatar base? Are you sure? y/n
echo ============================================================
:PROMPT
SET /P AREYOUSURE=Are you sure (Y/[N])?
IF /I "%AREYOUSURE%" NEQ "Y" GOTO END
rem List of folders to mirror
set "FOLDERS=Plugins\AvatarCore_AI Plugins\AvatarCore_Shared Plugins\AvatarCore_Manager Plugins\AvatarCore_MetaHuman Plugins\AvatarCore_STT Plugins\AvatarCore_TTS Plugins\BLogger Plugins\BSettings Plugins\BTools Plugins\RuntimeMetaHumanLipSync_5.6 Content\Project"
set "ANY_FAIL=0"
for %%F in (%FOLDERS%) do (
set "SRC=%SRC_ROOT%\%%F"
set "DST=%DST_ROOT%\%%F"
echo ------------------------------------------------------------
echo Mirroring: %%F
echo FROM: !SRC!
echo TO: !DST!
echo ------------------------------------------------------------
if not exist "!SRC!\" (
echo WARNING: Source folder missing, skipping: !SRC!
) else (
if not exist "!DST!\" (
mkdir "!DST!" 2>nul
)
robocopy "!SRC!" "!DST!" /MIR /DCOPY:DAT /COPY:DAT /R:2 /W:1 /FFT /Z /NP /FP /TS /TEE
set "RC=!ERRORLEVEL!"
echo Robocopy exit code: !RC!
if !RC! GEQ 8 (
echo ERROR: Robocopy reported failure for %%F ^(exit code !RC!^)
set "ANY_FAIL=1"
)
)
echo.
)
if "!ANY_FAIL!"=="1" (
echo ============================================================
echo Sync failed!
echo ============================================================
pause
exit /b 8
)
echo ============================================================
echo Sync finished.
echo ============================================================
:END
endlocal
pause
exit /b 0
Loading…
Cancel
Save