From ee8f4cf3a342988e65c8529931cdeb0a2511609f Mon Sep 17 00:00:00 2001 From: Tillman Staffen Date: Wed, 18 Mar 2026 16:24:15 +0100 Subject: [PATCH] Improved Stresstest, removed WebRTC EchoCancellation when Debug Play FIle, Added Environment --- .../DA_Mode_SPIE_SpieInnovationDay.uasset | 4 +- ...SpieInnovationDay_Instructions.schema.json | 8 ++ .../Widgets/Debug/W_StressTester.uasset | 4 +- .../Private/Recorder/STTRecorderDebugFile.cpp | 8 +- .../AvatarCore_STT/Private/STTManagerBase.cpp | 85 +++++++++++++++++-- .../Public/Recorder/STTRecorderDebugFile.h | 3 + .../AvatarCore_STT/Public/STTManagerBase.h | 3 + 7 files changed, 105 insertions(+), 10 deletions(-) diff --git a/Unreal/Content/SPIE/BP/Mode/DA_Mode_SPIE_SpieInnovationDay.uasset b/Unreal/Content/SPIE/BP/Mode/DA_Mode_SPIE_SpieInnovationDay.uasset index 5be0e5d..018e0c2 100644 --- a/Unreal/Content/SPIE/BP/Mode/DA_Mode_SPIE_SpieInnovationDay.uasset +++ b/Unreal/Content/SPIE/BP/Mode/DA_Mode_SPIE_SpieInnovationDay.uasset @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:355abbc862fed30bb427b9a3841e86ca393573111716c077afbcaba74e78b093 -size 6662 +oid sha256:9a5e4d3df8cb5e11085d0c2aba02f1ce3225747bdfab7e7949ab8d2fc75eefcf +size 6943 diff --git a/Unreal/Content/Schema/SystemInstructions/Mode_DA_Mode_SPIE_SpieInnovationDay_Instructions.schema.json b/Unreal/Content/Schema/SystemInstructions/Mode_DA_Mode_SPIE_SpieInnovationDay_Instructions.schema.json index a015610..deb2816 100644 --- a/Unreal/Content/Schema/SystemInstructions/Mode_DA_Mode_SPIE_SpieInnovationDay_Instructions.schema.json +++ b/Unreal/Content/Schema/SystemInstructions/Mode_DA_Mode_SPIE_SpieInnovationDay_Instructions.schema.json @@ -69,6 +69,14 @@ { "Name": "User Options", "Instruction": "The user has no option or UI or menu to adjust settings or anything else, only the developers of the Avatar app can do that. Politely refer back to answering questions the user has." + }, + { + "Name": "Numbers", + "Instruction": "Always spell out numbers. 12:10 Uhr = Zwölf Uhr zehn | 2001 = Zweitausend eins | 15. Mai = Fünfzehnter Mai" + }, + { + "Name": "Environment", + "Instruction": "You are in a futurisc, abstract environment depicting a wind turbine, a cell tower, a power pole and a city, there are glowing lines, a beautiful slightly clouded sky and some grass patches." } ] } diff --git a/Unreal/Plugins/AvatarCore_Manager/Content/Widgets/Debug/W_StressTester.uasset b/Unreal/Plugins/AvatarCore_Manager/Content/Widgets/Debug/W_StressTester.uasset index 11b16c4..bf3607f 100644 --- a/Unreal/Plugins/AvatarCore_Manager/Content/Widgets/Debug/W_StressTester.uasset +++ b/Unreal/Plugins/AvatarCore_Manager/Content/Widgets/Debug/W_StressTester.uasset @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:04e79d1add28bd85caa3891316f3389aa739cfd67c92adf5444b63feaa53f38d -size 418555 +oid sha256:f955efa79f384bb3a4cf64c7d46c0d0d141b516eea02312ecc4a110b3f84549f +size 427801 diff --git a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Recorder/STTRecorderDebugFile.cpp b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Recorder/STTRecorderDebugFile.cpp index f24ee48..ee78b56 100644 --- a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Recorder/STTRecorderDebugFile.cpp +++ b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Recorder/STTRecorderDebugFile.cpp @@ -155,7 +155,6 @@ void USTTRecorderDebugFile::DebugPlayAudioFile(FString FilePath) } STTManager->OnSTTFakeButtonStateChanged.Broadcast(true); - GetWorld()->GetTimerManager().SetTimer( PlaybackTimerHandle, // handle to cancel timer at a later time this, // the owning object @@ -166,6 +165,13 @@ void USTTRecorderDebugFile::DebugPlayAudioFile(FString FilePath) USTTRecorderDebugFile::SendChunk(); } +void USTTRecorderDebugFile::DebugClearAudioFile() +{ + STTManager->OnSTTFakeButtonStateChanged.Broadcast(false); + AudioComponent->Stop(); + GetWorld()->GetTimerManager().ClearTimer(PlaybackTimerHandle); +} + void USTTRecorderDebugFile::SendChunk() { // Send the PCM data in chunks matching the timer interval (0.03s) diff --git a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/STTManagerBase.cpp b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/STTManagerBase.cpp index b7399eb..3901500 100644 --- a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/STTManagerBase.cpp +++ b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/STTManagerBase.cpp @@ -2,6 +2,7 @@ #include "STTManagerBase.h" #include "Recorder/STTRecorderDebugFile.h" +#include "Preprocessor/STTPreprocessorWebRTC.h" #include "Kismet/GameplayStatics.h" void USTTManagerBase::InitSTTManager(bool AutoInitModules, USTTBaseProcessorConfig* InProcessorConfig, bool DebugMode) @@ -381,6 +382,12 @@ void USTTManagerBase::DebugPlayFileAsSTT(FString File) FileRecorder->DebugPlayAudioFile(File); } +void USTTManagerBase::DebugClearFilePlayback() +{ + USTTRecorderDebugFile* FileRecorder = DebugGetFileRecorder(); + FileRecorder->DebugClearAudioFile(); +} + USTTRecorderDebugFile* USTTManagerBase::DebugGetFileRecorder() { @@ -394,6 +401,44 @@ USTTRecorderDebugFile* USTTManagerBase::DebugGetFileRecorder() FileRecorder->InitSTTRecorder(this, bDebugMode); FileRecorder->OnChunkReceived.BindUFunction(STTPreprocessors[0], "OnChunkReceived"); STTRecorder = FileRecorder; + + // Disable echo cancellation on any WebRTC preprocessor in the chain + // (file playback has no real mic signal, so AEC would corrupt the audio) + for (int32 i = 0; i < STTPreprocessors.Num(); i++) + { + USTTPreprocessorWebRTC* WebRTCPP = Cast(STTPreprocessors[i]); + if (!WebRTCPP) + continue; + + UClass* PPClass = WebRTCPP->GetClass(); + + // Unbind outgoing delegate and destroy old instance + WebRTCPP->OnChunkProcessed.Unbind(); + WebRTCPP->DestroySTTPreprocessor(); + WebRTCPP->RemoveFromRoot(); + + // Create a replacement with echo_canceller disabled + FSTTBaseSettings ModifiedSettings = ProcessorConfig->BaseSettings; + ModifiedSettings.WebRTCSettings.echo_canceller = false; + + USTTPreprocessorWebRTC* NewWebRTCPP = NewObject(this, PPClass); + NewWebRTCPP->AddToRoot(); + NewWebRTCPP->InitSTTPreprocessor(this, ModifiedSettings, bDebugMode); + + // Re-bind incoming link + if (i == 0) + STTRecorder->OnChunkReceived.BindUFunction(NewWebRTCPP, "OnChunkReceived"); + else + STTPreprocessors[i - 1]->OnChunkProcessed.BindUFunction(NewWebRTCPP, "OnChunkReceived"); + + // Re-bind outgoing link + if (i == STTPreprocessors.Num() - 1) + NewWebRTCPP->OnChunkProcessed.BindUFunction(STTProcessor, "OnChunkReceived"); + else + NewWebRTCPP->OnChunkProcessed.BindUFunction(STTPreprocessors[i + 1], "OnChunkReceived"); + + STTPreprocessors[i] = NewWebRTCPP; + } } return FileRecorder; } @@ -402,11 +447,41 @@ void USTTManagerBase::DebugResetRecorder() { STTRecorder->DestroySTTRecorder(); STTRecorder->RemoveFromRoot(); - USTTRecorderBase* FileRecorder = NewObject(this, STTRecorderClass); - FileRecorder->AddToRoot(); - FileRecorder->InitSTTRecorder(this, bDebugMode); - FileRecorder->OnChunkReceived.BindUFunction(STTPreprocessors[0], "OnChunkReceived"); - STTRecorder = FileRecorder; + USTTRecorderBase* NewRecorder = NewObject(this, STTRecorderClass); + NewRecorder->AddToRoot(); + NewRecorder->InitSTTRecorder(this, bDebugMode); + NewRecorder->OnChunkReceived.BindUFunction(STTPreprocessors[0], "OnChunkReceived"); + STTRecorder = NewRecorder; + + // Restore echo cancellation on any WebRTC preprocessor using original config + for (int32 i = 0; i < STTPreprocessors.Num(); i++) + { + USTTPreprocessorWebRTC* WebRTCPP = Cast(STTPreprocessors[i]); + if (!WebRTCPP) + continue; + + UClass* PPClass = WebRTCPP->GetClass(); + + WebRTCPP->OnChunkProcessed.Unbind(); + WebRTCPP->DestroySTTPreprocessor(); + WebRTCPP->RemoveFromRoot(); + + USTTPreprocessorWebRTC* NewWebRTCPP = NewObject(this, PPClass); + NewWebRTCPP->AddToRoot(); + NewWebRTCPP->InitSTTPreprocessor(this, ProcessorConfig->BaseSettings, bDebugMode); + + if (i == 0) + STTRecorder->OnChunkReceived.BindUFunction(NewWebRTCPP, "OnChunkReceived"); + else + STTPreprocessors[i - 1]->OnChunkProcessed.BindUFunction(NewWebRTCPP, "OnChunkReceived"); + + if (i == STTPreprocessors.Num() - 1) + NewWebRTCPP->OnChunkProcessed.BindUFunction(STTProcessor, "OnChunkReceived"); + else + NewWebRTCPP->OnChunkProcessed.BindUFunction(STTPreprocessors[i + 1], "OnChunkReceived"); + + STTPreprocessors[i] = NewWebRTCPP; + } } bool USTTManagerBase::IsSTTFullyInitialized() diff --git a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Recorder/STTRecorderDebugFile.h b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Recorder/STTRecorderDebugFile.h index 8e27ca5..529b5c4 100644 --- a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Recorder/STTRecorderDebugFile.h +++ b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Recorder/STTRecorderDebugFile.h @@ -29,6 +29,9 @@ public: UFUNCTION(BlueprintCallable, Category = STTManager) void DebugPlayAudioFile(FString FilePath); + + UFUNCTION(BlueprintCallable, Category = STTManager) + void DebugClearAudioFile(); private: diff --git a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/STTManagerBase.h b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/STTManagerBase.h index 36bcdff..cdf076b 100644 --- a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/STTManagerBase.h +++ b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/STTManagerBase.h @@ -168,6 +168,9 @@ public: UFUNCTION(BlueprintCallable, Category = "AvatarCoreSTT") void DebugPlayFileAsSTT(FString File); + UFUNCTION(BlueprintCallable, Category = "AvatarCoreSTT") + void DebugClearFilePlayback(); + USTTRecorderDebugFile* DebugGetFileRecorder(); UFUNCTION(BlueprintCallable, Category = "AvatarCoreSTT")