Matched with base to remove enum dependancy

Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/AIBaseManager.cpp

@@ -17,6 +17,8 @@ void UAIBaseManager::InitAIManager(UAIBaseConfig* AIConfig, bool DebugMode, AAct
 	CurrentConfig = AIConfig;
     CurrentConfig->AddToRoot();
 
+    UAIBaseManager::AddRepeatSystemInstruction();
+
     // Store the reference actor for world context
     WorldReferenceActor = InWorldReferenceActor;
 
@@ -122,7 +124,7 @@ void UAIBaseManager::SetNewState(EAvatarCoreAIState NewState, bool ForceState)
     }
 }
 
-void UAIBaseManager::SendResponse(const FString& Response, bool NotifyDelay = false, FString OverrideInstruction = "")
+void UAIBaseManager::SendResponse(const FString& Response, bool NotifyDelay = false, bool TriggerResponse = true)
 {
     AnswerCache.Empty();
     ResponseID++;
@@ -144,10 +146,10 @@ void UAIBaseManager::RepeatText(FString TextToRepeat, bool DoRephrase)
     ResponseID++;
 	FString Instruction;
 	if (DoRephrase)
-		Instruction = "Repeat the text in your own words."; 
+		Instruction = "Repeat the text in your own words: " + TextToRepeat;
 	else
-		Instruction = "Repeat the text exactly word for word.";
+		Instruction = "[REPEAT] " + TextToRepeat;
-	SendResponse(TextToRepeat, false, Instruction);
+	SendResponse(Instruction, false, true);
 }
 
 void UAIBaseManager::ClearAI()
@@ -378,36 +380,100 @@ void UAIBaseManager::OnAIResponse(const FString& Chunk, bool IsFinal)
     }
 }
 
-void UAIBaseManager::AddSystemInstruction(const FName Name, const FString NewSystemInstruction)
+void UAIBaseManager::AddSystemInstruction(const FName Name, const FString NewSystemInstruction, bool AddAsFirst = false)
 {
     UAIBaseManager::RemoveSystemInstruction(Name);
     FSystemInstruction tmpEntry;
     tmpEntry.Name = Name;
     tmpEntry.Instruction = NewSystemInstruction;
-    CurrentConfig->SystemPromps.Add(tmpEntry);
+    
+    if(AddAsFirst)
+    {
+        TArray<FSystemInstruction> tmpSystemPrompts;
+        tmpSystemPrompts.Add(tmpEntry);
+        tmpSystemPrompts.Append(CurrentConfig->SystemPrompts);
+        CurrentConfig->SystemPrompts = tmpSystemPrompts;
+    }
+    else
+        CurrentConfig->SystemPrompts.Add(tmpEntry);
+
     BroadcastAILog(FString::Printf(TEXT("AI Manager added System Instruction %s"), *Name.ToString()));
 }
 
 
 void UAIBaseManager::ClearAllSystemInstructios()
 {
-    CurrentConfig->SystemPromps.Empty();
+    CurrentConfig->SystemPrompts.Empty();
     BroadcastAILog(FString::Printf(TEXT("AI Manager wiped all System Instructions")));
+    UAIBaseManager::AddRepeatSystemInstruction();
+}
+
+void UAIBaseManager::AddRepeatSystemInstruction()
+{
+    UAIBaseManager::AddSystemInstruction(TEXT("Repeat Text"), TEXT("If the text starts with [REPEAT], repeat the text exactly word for word."), true);
 }
 
 void UAIBaseManager::RemoveSystemInstruction(const FName Name)
 {
     // Iterate in reverse to safely remove while iterating
-    for (int32 i = CurrentConfig->SystemPromps.Num() - 1; i >= 0; --i)
+    for (int32 i = CurrentConfig->SystemPrompts.Num() - 1; i >= 0; --i)
     {
-        if (CurrentConfig->SystemPromps[i].Name == Name)
+        if (CurrentConfig->SystemPrompts[i].Name == Name)
         {
-            CurrentConfig->SystemPromps.RemoveAt(i);
+            CurrentConfig->SystemPrompts.RemoveAt(i);
             BroadcastAILog(FString::Printf(TEXT("AI Manager removed System Instruction %s"), *Name.ToString()));
         }
     }
 }
 
+FString UAIBaseManager::GetSystemInstructionPromptString(bool AsJsonString = false)
+{
+    FString prompt;
+    
+    if(AsJsonString)
+    {
+        TArray<TSharedPtr<FJsonValue>> JsonArray;
+
+        for (const FSystemInstruction& Item : CurrentConfig->SystemPrompts)
+        {
+            if (!Item.Instruction.IsEmpty())
+            {
+                // Each entry: { "Name": "Instruction" }
+                TSharedPtr<FJsonObject> Obj = MakeShared<FJsonObject>();
+                Obj->SetStringField(Item.Name.ToString(), Item.Instruction);
+                JsonArray.Add(MakeShared<FJsonValueObject>(Obj));
+            }
+        }
+
+        // Add a fallback if no entries exist
+        if (JsonArray.Num() == 0)
+        {
+            TSharedPtr<FJsonObject> DefaultObj = MakeShared<FJsonObject>();
+            DefaultObj->SetStringField(TEXT("Default"), TEXT("Just do exactly what the user wants you to do."));
+            JsonArray.Add(MakeShared<FJsonValueObject>(DefaultObj));
+        }
+
+        // 2) Serialize array into a single JSON string
+        TSharedRef<TJsonWriter<>> Writer = TJsonWriterFactory<>::Create(&prompt);
+        FJsonSerializer::Serialize(JsonArray, Writer);
+    }
+    else
+    {
+        for (int32 Index = 0; Index < CurrentConfig->SystemPrompts.Num(); ++Index)
+        {
+            FSystemInstruction& Item = CurrentConfig->SystemPrompts[Index];
+            if (!Item.Instruction.IsEmpty())
+            {
+                prompt += "# " + Item.Name.ToString() + TEXT("\r\n");
+                prompt += Item.Instruction;
+                if(Index < CurrentConfig->SystemPrompts.Num() - 1)
+                    prompt += TEXT("\r\n\r\n");
+            }
+        }
+    }
+    return prompt;
+}
+
 void UAIBaseManager::ResetRequestTimeout()
 {
     UAIBaseManager::ClearRequestTimeout();

Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/RealtimeAPI/AvatarCoreAIRealtime.cpp

@@ -37,15 +37,10 @@ void UAvatarCoreAIRealtime::DeactivateAI()
     UAIBaseManager::DeactivateAI();
 }
 
-void UAvatarCoreAIRealtime::SendResponse(const FString& Response, bool NotifyDelay, FString OverrideInstruction)
+void UAvatarCoreAIRealtime::SendResponse(const FString& Response, bool NotifyDelay = false, bool TriggerResponse = true)
 {
-    UAIBaseManager::SendResponse(Response, NotifyDelay, OverrideInstruction);
+    UAIBaseManager::SendResponse(Response, NotifyDelay, TriggerResponse);
-    FString RequestWithInstruction;
+    UAvatarCoreAIRealtime::CreateConversationItem(Response, EOpenAIRoleType::User, TriggerResponse);
-    if (OverrideInstruction.IsEmpty())
-        RequestWithInstruction = Response;
-    else
-        RequestWithInstruction = OverrideInstruction + ": " + Response;
-    UAvatarCoreAIRealtime::CreateConversationItem(RequestWithInstruction, EOpenAIRoleType::User, true);
 }
 
 void UAvatarCoreAIRealtime::ClearAI()
@@ -73,35 +68,7 @@ void UAvatarCoreAIRealtime::UpdateSession()
     // Build session object
     TSharedPtr<FJsonObject> SessionObj = MakeShareable(new FJsonObject);
 
-    FString InstructionsJsonString;
+    FString InstructionsString = UAIBaseManager::GetSystemInstructionPromptString(false);
-
-    // 1) Build an array of JSON objects, each having one field: Name -> Instruction
-    {
-        TArray<TSharedPtr<FJsonValue>> JsonArray;
-
-        for (const FSystemInstruction& Item : CurrentConfig->SystemPromps)
-        {
-            if (!Item.Instruction.IsEmpty())
-            {
-                // Each entry: { "Name": "Instruction" }
-                TSharedPtr<FJsonObject> Obj = MakeShared<FJsonObject>();
-                Obj->SetStringField(Item.Name.ToString(), Item.Instruction);
-                JsonArray.Add(MakeShared<FJsonValueObject>(Obj));
-            }
-        }
-
-        // Add a fallback if no entries exist
-        if (JsonArray.Num() == 0)
-        {
-            TSharedPtr<FJsonObject> DefaultObj = MakeShared<FJsonObject>();
-            DefaultObj->SetStringField(TEXT("Default"), TEXT("Just do exactly what the user wants you to do."));
-            JsonArray.Add(MakeShared<FJsonValueObject>(DefaultObj));
-        }
-
-        // 2) Serialize array into a single JSON string
-        TSharedRef<TJsonWriter<>> Writer = TJsonWriterFactory<>::Create(&InstructionsJsonString);
-        FJsonSerializer::Serialize(JsonArray, Writer);
-    }
 
     if (RealtimeConfig->AIModelAudioOutput)
         Modalities.Add(TEXT("audio"));
@@ -114,7 +81,7 @@ void UAvatarCoreAIRealtime::UpdateSession()
         }());
 
     // 3) Store serialized JSON array as a *string* field in SessionObj
-    SessionObj->SetStringField(TEXT("instructions"), InstructionsJsonString);
+    SessionObj->SetStringField(TEXT("instructions"), InstructionsString);
 
     SessionObj->SetStringField(TEXT("type"), TEXT("realtime"));
 

Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/AIBaseConfig.h

@@ -26,7 +26,7 @@ public:
 
 	// All those neat little system prompts that make our avatars sooo great
 	UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Settings", meta = (ExposeOnSpawn = "true"))
-	TArray<FSystemInstruction> SystemPromps;
+	TArray<FSystemInstruction> SystemPrompts;
 
 	UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|MCP", meta = (ExposeOnSpawn = "true"))
 	bool bUseMCPServer = true;

Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/AIBaseManager.h

@@ -97,7 +97,7 @@ public:
     * Send Response/Question to the AI Model. If NotifyDelay is true call the DelayedAnswer Event when time defined in AIConfig has passed.
     */
     UFUNCTION(BlueprintCallable, Category = "AvatarCoreAI")
-    virtual void SendResponse(const FString& Response, bool NotifyDelay, FString OverrideInstruction);
+    virtual void SendResponse(const FString& Response, bool NotifyDelay, bool TriggerResponse);
 
     /**
     * Make the AI Model repeat the Text.
@@ -132,21 +132,31 @@ public:
     /**
     * Add a new system instruction by name.
     */
-	UFUNCTION(BlueprintCallable, Category = "AvatarCoreAI")
+	UFUNCTION(BlueprintCallable, Category = "AvatarCoreAI|System Instruction")
-    void AddSystemInstruction(const FName Name, const FString NewSystemInstruction);
+    void AddSystemInstruction(const FName Name, const FString NewSystemInstruction, bool AddAsFirst);
 
     /**
     * Remove a system instruction by name.
     */
-    UFUNCTION(BlueprintCallable, Category = "AvatarCoreAI")
+    UFUNCTION(BlueprintCallable, Category = "AvatarCoreAI|System Instruction")
     void RemoveSystemInstruction(const FName Name);
 
+    /**
+    * Parse to System Prompt
+    */
+    UFUNCTION(BlueprintCallable, Category = "AvatarCoreAI|System Instruction")
+    FString GetSystemInstructionPromptString(bool AsJsonString);
+
     /**
     * Clear all System Instruction.
     */
-    UFUNCTION(BlueprintCallable, Category = "AvatarCoreAI")
+    UFUNCTION(BlueprintCallable, Category = "AvatarCoreAI|System Instruction")
     void ClearAllSystemInstructios();
 
+    // Add the prompt that let the avatar repeat what we want it to say
+    UFUNCTION(BlueprintCallable, Category = "AvatarCoreAI|System Instruction")
+    void AddRepeatSystemInstruction();
+
     /**
     * Timeout Handling
     */

Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/RealtimeAPI/AvatarCoreAIRealtime.h

@@ -156,7 +156,7 @@ public:
 	void ActivateAI() override;
 	void DeactivateAI() override;
 	void UpdateSession() override;
-	void SendResponse(const FString& Response, bool NotifyDelay = false, FString OverrideInstruction = "") override;
+	void SendResponse(const FString& Response, bool NotifyDelay, bool TriggerResponse) override;
 	void ClearAI() override;
 
 	void ConnectToWebSocket();

Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Processor/Azure/AzureRunnable.cpp

@@ -79,10 +79,25 @@ bool FAzureRunnable::Init()
     // Bind Recognized event for final results
     Recognizer->Recognized.Connect([WeakOwner](const auto& EventArgs) {
         FString RecognizedText = UTF8_TO_TCHAR(EventArgs.Result->Text.c_str());
-        AsyncTask(ENamedThreads::GameThread, [WeakOwner, RecognizedText]() {
+        FString DetectedLangUE = TEXT("LANGUAGE_NOT_DETECTED");
+
+        try
+        {
+            auto LangResult = SpeechSDK::AutoDetectSourceLanguageResult::FromResult(EventArgs.Result);
+            if (LangResult && !LangResult->Language.empty())
+            {
+                DetectedLangUE = UTF8_TO_TCHAR(LangResult->Language.c_str());
+            }
+        }
+        catch (...)
+        {
+            // Keep default LANGUAGE_NOT_DETECTED
+        }
+
+        AsyncTask(ENamedThreads::GameThread, [WeakOwner, RecognizedText, DetectedLangUE]() {
             if (WeakOwner.IsValid() && WeakOwner.Get() != nullptr)
             {
-                WeakOwner->OnRecognized(RecognizedText);
+                WeakOwner->OnRecognized(RecognizedText, DetectedLangUE);
             }
         });
     });

Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Processor/Azure/STTProcessorAzure.cpp

@@ -148,11 +148,13 @@ void USTTProcessorAzure::OnRecognizing(const FString& RecognizedText)
         USTTProcessorBase::OnTranscriptionIntermediateResult(TranscriptionCounter, *intermediateResult);
 }
 
-void USTTProcessorAzure::OnRecognized(const FString& RecognizedText)
+void USTTProcessorAzure::OnRecognized(const FString& RecognizedText, const FString& Language)
 {
     if (IsValid(STTManager) && STTManager->IsBlocked())
         return;
     
+    this->DetectedLanguage = Language;
+
     if (!intermediateResult.IsEmpty())
         intermediateResult += " " + RecognizedText;
     else
@@ -161,7 +163,7 @@ void USTTProcessorAzure::OnRecognized(const FString& RecognizedText)
         USTTProcessorBase::OnTranscriptionIntermediateResult(TranscriptionCounter, *intermediateResult);
     }
     else {
-        USTTProcessorBase::OnTranscriptionResult(TranscriptionCounter, *intermediateResult);
+        USTTProcessorBase::OnTranscriptionResult(TranscriptionCounter, *intermediateResult, this->DetectedLanguage);
         intermediateResult.Empty();       
     }
 }
@@ -179,7 +181,7 @@ void USTTProcessorAzure::OnRunnableEnded()
     bTranscriptionRunning = false;
 
     if (!intermediateResult.IsEmpty()) {
-        STTManager->OnTranscriptionReceived.Broadcast(TranscriptionCounter, *intermediateResult);
+        STTManager->OnTranscriptionReceived.Broadcast(TranscriptionCounter, *intermediateResult, this->DetectedLanguage);
         intermediateResult.Empty();
     }
 

Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Processor/STTProcessorBase.cpp

@@ -11,13 +11,13 @@ bool USTTProcessorBase::IsTranscriptionRunning()
 	return bTranscriptionRunning;
 }
 
-void USTTProcessorBase::OnTranscriptionResult(int32 TranscriptionID, FString TranscriptionResult)
+void USTTProcessorBase::OnTranscriptionResult(int32 TranscriptionID, FString TranscriptionResult, FString Language)
 {
 	bTranscriptionRunning = false;
 	if (IsValid(STTManager))
 	{
 		STTManager->UserSpeechStateChanged(ESTTTalkingState::SILENCE);
-		STTManager->SendTranscription(TranscriptionID, TranscriptionResult);
+		STTManager->SendTranscription(TranscriptionID, TranscriptionResult, Language);
 	}
 }
 

Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/Processor/Whisper/STTProcessorWhisper.cpp

@@ -269,10 +269,19 @@ void USTTProcessorWhisper::BuildMultipartBody(const TArray<uint8>& WavData, cons
 	AppendStringToBody(OutBody, TEXT("Content-Disposition: form-data; name=\"model\"\r\n\r\n"));
 	AppendStringToBody(OutBody, TranscribeModelEnumToString(WhisperProcessorConfig->Model) + TEXT("\r\n"));
 
+	if (WhisperProcessorConfig->Model == EOpenAITranscriptionModel::Whisper1)
+	{
+		BoundaryLine = FString::Printf(TEXT("--%s\r\n"), *Boundary);
+		AppendStringToBody(OutBody, BoundaryLine);
+		AppendStringToBody(OutBody, TEXT("Content-Disposition: form-data; name=\"response_format\"\r\n\r\n"));
+		AppendStringToBody(OutBody, TEXT("verbose_json\r\n"));
+	}
+
 	if (!Prompt.IsEmpty())
 	{
 		BoundaryLine = FString::Printf(TEXT("--%s\r\n"), *Boundary);
 		AppendStringToBody(OutBody, BoundaryLine);
+
 		AppendStringToBody(OutBody, TEXT("Content-Disposition: form-data; name=\"prompt\"\r\n\r\n"));
 		AppendStringToBody(OutBody, Prompt + TEXT("\r\n"));
 	}
@@ -432,7 +441,15 @@ void USTTProcessorWhisper::SendWhisperRequest(TArray<uint8>&& WavData)
 			return;
 		}
 
-		Self->OnTranscriptionResult(TranscriptionId, Text);
+		UE_LOG(LogTemp, Warning, TEXT("OpenAI says:  %s"), *JsonString);
+
+		FString Language;
+		if (RootObject->TryGetStringField(TEXT("language"), Language) && !Language.IsEmpty())
+		{
+			Self->DetectedLanguage = Language;
+		}
+
+		Self->OnTranscriptionResult(TranscriptionId, Text, Self->DetectedLanguage);
 	});
 
 	ActiveRequests.Add(RequestPtr);

Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/STTManagerBase.cpp

@@ -226,9 +226,9 @@ float USTTManagerBase::GetRemainingTalkingTimeFactor()
 		return 1;
 }
 
-void USTTManagerBase::DebugSTTInput(FString DebugSentence)
+void USTTManagerBase::DebugSTTInput(FString DebugSentence, FString Language = "de")
 {
-	OnTranscriptionReceived.Broadcast(0, DebugSentence);
+	OnTranscriptionReceived.Broadcast(0, DebugSentence, Language);
 }
 
 void USTTManagerBase::PTTStateChanged(bool BtnPressed)
@@ -325,9 +325,9 @@ void USTTManagerBase::SendTranscriptionChunk(uint32 TranscriptionID, FString con
 	OnTranscriptionChunkReceived.Broadcast(TranscriptionID, USTTManagerBase::CheckForWordReplacement(content));
 }
 
-void USTTManagerBase::SendTranscription(uint32 TranscriptionID, FString content)
+void USTTManagerBase::SendTranscription(uint32 TranscriptionID, FString content, FString Language)
 {
-	OnTranscriptionReceived.Broadcast(TranscriptionID, USTTManagerBase::CheckForWordReplacement(content));
+	OnTranscriptionReceived.Broadcast(TranscriptionID, USTTManagerBase::CheckForWordReplacement(content), Language);
 }
 
 void USTTManagerBase::SendUIStateChanged()

Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Processor/Azure/STTProcessorAzure.h

@@ -52,7 +52,7 @@ private:
 
 public:
 	void OnRecognizing(const FString& RecognizedText);
-	void OnRecognized(const FString& RecognizedText);
+	void OnRecognized(const FString& RecognizedText, const FString& Language);
 
 	UFUNCTION()
 	void OnConnectionSuccess();

Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/Processor/STTProcessorBase.h

@@ -33,7 +33,7 @@ public:
 	bool IsTranscriptionRunning();
 
 	UFUNCTION()
-	void OnTranscriptionResult(int32 TranscriptionID, FString TranscriptionResult);
+	void OnTranscriptionResult(int32 TranscriptionID, FString TranscriptionResult, FString Language);
 
 	UFUNCTION()
 	void OnTranscriptionIntermediateResult(int32 TranscriptionID, FString TranscriptionIntermediateResult);
@@ -52,4 +52,6 @@ protected:
 	bool bTranscriptionRunning = false;
 	int32 TranscriptionCounter = 0;
 
+	FString DetectedLanguage = "LANGUAGE_NOT_DETECTED";
+
 };

Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/STTManagerBase.h

@@ -13,7 +13,7 @@
 
 
 DECLARE_DYNAMIC_MULTICAST_DELEGATE_TwoParams(FMulticastDelegateTranscriptionChunkReceived, int32, TranscribeID, FString, Content);
-DECLARE_DYNAMIC_MULTICAST_DELEGATE_TwoParams(FMulticastDelegateTranscriptionReceived, int32, TranscribeID, FString, Content);
+DECLARE_DYNAMIC_MULTICAST_DELEGATE_ThreeParams(FMulticastDelegateTranscriptionReceived, int32, TranscribeID, FString, Content, FString, Language);
 DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FMulticastDelegateSpeechStateChanged, ESTTTalkingState, TalkingState);
 DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FMulticastDelegateSpeechStateChangedForUI, ESTTTalkingState, TalkingState);
 DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FMulticastDelegateSTTBlocked, bool, IsBlocked);
@@ -119,7 +119,7 @@ public:
 	float GetRemainingTalkingTimeFactor();
 
 	UFUNCTION(BlueprintCallable, Category = "AvatarCoreSTT")
-	void DebugSTTInput(FString DebugSentence);
+	void DebugSTTInput(FString DebugSentence, FString Language);
 
 	UFUNCTION(BlueprintCallable, Category = "AvatarCoreSTT")
 	void PTTStateChanged(bool BtnPressed);
@@ -143,7 +143,7 @@ public:
 	void SendTranscriptionChunk(uint32 TranscriptionID, FString content);
 
 	UFUNCTION()
-	void SendTranscription(uint32 TranscriptionID, FString content);
+	void SendTranscription(uint32 TranscriptionID, FString content, FString Language);
 
 	UFUNCTION()
 	void SendUIStateChanged();