Various improvements to make the stt connection etter, added a stresstester widget, activated background audio

Unreal/Config/DefaultEngine.ini

@@ -119,3 +119,6 @@ bSupportUVFromHitResults=True
 SettingsStruct=/Game/SPIE/BP/S_SPIE_ConfigSettings.S_SPIE_ConfigSettings
 FileName=Spie_Config
 
+[/Script/IOSRuntimeSettings.IOSRuntimeSettings]
+bSupportsBackgroundAudio=True
+

Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Processor/Azure/STTProcessorAzure.cpp

@@ -200,7 +200,8 @@ void USTTProcessorAzure::OnRunnableEnded()
         else {
             // Ensure we return to SILENCE even if no result was produced
             // (empty audio, network timeout, etc.) to prevent stuck TRANSCRIBING state
-            STTManager->UserSpeechStateChanged(ESTTTalkingState::SILENCE);
+            if(!STTManager->IsBlocked())
+                STTManager->UserSpeechStateChanged(ESTTTalkingState::SILENCE);
         }
     }
 }

Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Recorder/STTRecorderDebugFile.cpp

@@ -129,17 +129,12 @@ void USTTRecorderDebugFile::DebugPlayAudioFile(FString FilePath)
         STTManager->OnSTTError.Broadcast(FString::Printf(TEXT("Failed to parse WAV file: %s"), *FilePath));
         return;
     }
-
-    TArray<int16> SilenceChunks;
-    SilenceChunks.Init(0, AudioInformation.SampleRate * AudioInformation.NumChannels * 4);
-
-    PCMData.Append(SilenceChunks);
-
     STTManager->OnSTTLog.Broadcast(FString::Printf(TEXT("Playing Soundfile %s with Sample Rate: %i and %i channels"), *FilePath, AudioInformation.SampleRate, AudioInformation.NumChannels));
     
     AudioComponent->Stop();
     ProceduralSoundWave->SetSampleRate(AudioInformation.SampleRate); // Set the sample rate, e.g., 44.1 kHz
     ProceduralSoundWave->NumChannels = AudioInformation.NumChannels;
+
     AudioComponent->SetSound(ProceduralSoundWave);
 
     if (ProceduralSoundWave && PCMData.Num() > 0)
@@ -159,7 +154,7 @@ void USTTRecorderDebugFile::DebugPlayAudioFile(FString FilePath)
         GetWorld()->GetTimerManager().ClearTimer(PlaybackTimerHandle);
     }
 
-    STTManager->PTTStateChanged(true);
+    STTManager->OnSTTFakeButtonStateChanged.Broadcast(true);
 
     GetWorld()->GetTimerManager().SetTimer(
         PlaybackTimerHandle, // handle to cancel timer at a later time
@@ -173,18 +168,20 @@ void USTTRecorderDebugFile::DebugPlayAudioFile(FString FilePath)
 
 void USTTRecorderDebugFile::SendChunk()
 {
-    // Send the PCM data in chunks
-    int32 ChunkSize = (AudioInformation.SampleRate/0.031f); // Number of samples per chunk
+    // Send the PCM data in chunks matching the timer interval (0.03s)
+    int32 ChunkSize = FMath::CeilToInt32(AudioInformation.SampleRate * AudioInformation.NumChannels * 0.03f); // Samples for 30ms
     int32 RemainingSamples = FMath::Min(ChunkSize, PCMData.Num() - SentChunks);
-    TArray<int16> Chunk;
-    Chunk.Append(&PCMData[SentChunks], RemainingSamples);
-    OnChunkReceived.ExecuteIfBound(Chunk, AudioInformation);
-    SentChunks += ChunkSize;
-    if (SentChunks >= PCMData.Num()) {
+    if (RemainingSamples <= 0)
+    {
         GetWorld()->GetTimerManager().ClearTimer(PlaybackTimerHandle);
-        STTManager->PTTStateChanged(false);
+        STTManager->OnSTTFakeButtonStateChanged.Broadcast(false);
         STTManager->OnSTTLog.Broadcast(FString::Printf(TEXT("Finished sending %i audio chunks (SampleRate: %i Channels: %i)"), PCMData.Num(), AudioInformation.SampleRate, AudioInformation.NumChannels));
+        return;
     }
+    TArray<int16> Chunk;
+    Chunk.Append(&PCMData[SentChunks], RemainingSamples);
+    OnChunkReceived.ExecuteIfBound(Chunk, AudioInformation);
+    SentChunks += RemainingSamples;       
 }
 
 bool USTTRecorderDebugFile::ParseWavFile(const TArray<uint8>& FileData, TArray<int16>& OutPCMData, FAudioInformation& OutAudioInformation)

Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/STTManagerBase.cpp

@@ -147,6 +147,7 @@ void USTTManagerBase::ClearSTTManager()
 
 void USTTManagerBase::SetBlocked(bool isBlocked)
 {
+
 	if (!bIsInitialized)
 		return;
 
@@ -158,9 +159,12 @@ void USTTManagerBase::SetBlocked(bool isBlocked)
 			UserSpeechStateChanged(ESTTTalkingState::BLOCKED);
 		}	
 		else {
-			if (bDebugMode)
-				OnSTTLog.Broadcast("Set STT Unblocked");
-			UserSpeechStateChanged(ESTTTalkingState::SILENCE);
+			if (CurrentSpeechState == ESTTTalkingState::BLOCKED)
+			{
+				if (bDebugMode)
+					OnSTTLog.Broadcast("Set STT Unblocked");
+				UserSpeechStateChanged(ESTTTalkingState::SILENCE);
+			}
 		}
 			
 	}
@@ -365,23 +369,44 @@ void USTTManagerBase::SetPTTPostRollTime(float NewTime)
 	PTTPostRollTime = NewTime;
 }
 
-void USTTManagerBase::DebugSetFilePlayback(){
+void USTTManagerBase::DebugOpenFilePicker()
+{
+	USTTRecorderDebugFile* FileRecorder = DebugGetFileRecorder();
+	FileRecorder->DebugOpenFilepicker();
+}
+
+void USTTManagerBase::DebugPlayFileAsSTT(FString File)
+{
+	USTTRecorderDebugFile* FileRecorder = DebugGetFileRecorder();
+	FileRecorder->DebugPlayAudioFile(File);
+}
+
+USTTRecorderDebugFile* USTTManagerBase::DebugGetFileRecorder()
+{
 
 	USTTRecorderDebugFile* FileRecorder = Cast<USTTRecorderDebugFile>(STTRecorder);
-	if(FileRecorder)
-		FileRecorder->DebugOpenFilepicker();
-	else
+	if (!FileRecorder)
 	{
 		STTRecorder->DestroySTTRecorder();
 		STTRecorder->RemoveFromRoot();
-		USTTRecorderDebugFile* STTFileRecorder = NewObject<USTTRecorderDebugFile>(this);
-		STTFileRecorder->AddToRoot();
-		STTFileRecorder->InitSTTRecorder(this, bDebugMode);
-		STTFileRecorder->OnChunkReceived.BindUFunction(STTPreprocessors[0], "OnChunkReceived");
-		STTFileRecorder->DebugOpenFilepicker();
-		STTRecorder = STTFileRecorder;
+		FileRecorder = NewObject<USTTRecorderDebugFile>(this);
+		FileRecorder->AddToRoot();
+		FileRecorder->InitSTTRecorder(this, bDebugMode);
+		FileRecorder->OnChunkReceived.BindUFunction(STTPreprocessors[0], "OnChunkReceived");
+		STTRecorder = FileRecorder;
 	}
+	return FileRecorder;
+}
 
+void USTTManagerBase::DebugResetRecorder()
+{
+	STTRecorder->DestroySTTRecorder();
+	STTRecorder->RemoveFromRoot();
+	USTTRecorderBase* FileRecorder = NewObject<USTTRecorderBase>(this, STTRecorderClass);
+	FileRecorder->AddToRoot();
+	FileRecorder->InitSTTRecorder(this, bDebugMode);
+	FileRecorder->OnChunkReceived.BindUFunction(STTPreprocessors[0], "OnChunkReceived");
+	STTRecorder = FileRecorder;
 }
 
 bool USTTManagerBase::IsSTTFullyInitialized()

Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/STTManagerBase.h

@@ -17,11 +17,14 @@ DECLARE_DYNAMIC_MULTICAST_DELEGATE_ThreeParams(FMulticastDelegateTranscriptionRe
 DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FMulticastDelegateSpeechStateChanged, ESTTTalkingState, TalkingState);
 DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FMulticastDelegateSpeechStateChangedForUI, ESTTTalkingState, TalkingState);
 DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FMulticastDelegateSTTBlocked, bool, IsBlocked);
+DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FMulticastDelegateFakeButtonStateChanged, bool, IsPressed);
 DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FMulticastDelegateSTTLog, FString, LogContent);
 DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FMulticastDelegateSTTError, FString, LogError);
 DECLARE_DYNAMIC_MULTICAST_DELEGATE(FMulticastDelegateSpeechDetectedWhileBlocked);
 DECLARE_DYNAMIC_MULTICAST_DELEGATE(FMulticastDelegateIsReady);
 
+class USTTRecorderDebugFile;
+
 /**
  * 
  */
@@ -41,6 +44,8 @@ public:
 	UPROPERTY(BlueprintAssignable, Category = "AvatarCoreSTT")
 	FMulticastDelegateTranscriptionReceived OnTranscriptionReceived;
 	UPROPERTY(BlueprintAssignable, Category = "AvatarCoreSTT")
+	FMulticastDelegateFakeButtonStateChanged OnSTTFakeButtonStateChanged;
+	UPROPERTY(BlueprintAssignable, Category = "AvatarCoreSTT")
 	FMulticastDelegateSTTLog OnSTTLog;
 	UPROPERTY(BlueprintAssignable, Category = "AvatarCoreSTT")
 	FMulticastDelegateSTTError OnSTTError;
@@ -158,7 +163,15 @@ public:
 	void SetPTTPostRollTime(float NewTime);
 
 	UFUNCTION(BlueprintCallable, Category = "AvatarCoreSTT")
-	void DebugSetFilePlayback();
+	void DebugOpenFilePicker();
+
+	UFUNCTION(BlueprintCallable, Category = "AvatarCoreSTT")
+	void DebugPlayFileAsSTT(FString File);
+
+	USTTRecorderDebugFile* DebugGetFileRecorder();
+
+	UFUNCTION(BlueprintCallable, Category = "AvatarCoreSTT")
+	void DebugResetRecorder();
 
 	UFUNCTION(BlueprintCallable, Category = "AvatarCoreSTT")
 	bool IsSTTFullyInitialized();