[RECOMPILE] Matched with Base, now with refactored state system

Unreal/Plugins/AvatarCore_AI/AvatarCore_AI.uplugin

@@ -2,7 +2,7 @@
 	"FileVersion": 3,
 	"Version": 1,
 	"VersionName": "1.1",
-	"FriendlyName": "Avatar Core AI",
+	"FriendlyName": "AvatarCore AI",
 	"Description": "A wrapper for OpenAI Assistents",
 	"Category": "Other",
 	"CreatedBy": "b.RexGmbh",

Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/AIBaseManager.cpp

@@ -86,7 +86,7 @@ void UAIBaseManager::InitAIManagerChild(UAIBaseConfig* AIConfig, AActor* InWorld
 
 void UAIBaseManager::DeinitAIManager()
 {
-    //ClearAI();
+    PreviousMessages.Empty();
 
     if (MCPManager) {
         MCPManager->DeinitMCPManager();
@@ -123,12 +123,14 @@ void UAIBaseManager::DeinitAIManager()
 
 void UAIBaseManager::OnAIReady()
 {
+    OnAIActivationStateChanged.Broadcast(true);
+
     UAIBaseManager::SetNewState(EAvatarCoreAIState::Ready);
     if (!ResponseQueue.IsEmpty())
     {
-        FPrompt QueuedPrompt;
+        FAIMessage QueuedPrompt;
         ResponseQueue.Dequeue(QueuedPrompt);
-        UAIBaseManager::SendResponse(QueuedPrompt.Prompt, QueuedPrompt.Role, false, true);
+        UAIBaseManager::SendResponse(QueuedPrompt, false, true);
     }
 }
 
@@ -137,6 +139,12 @@ bool UAIBaseManager::IsAIActivated()
     return CurrentAIState > EAvatarCoreAIState::Initialized;
 }
 
+void UAIBaseManager::DeactivateAI()
+{
+    PreviousMessages.Empty();
+    OnAIActivationStateChanged.Broadcast(false);
+}
+
 void UAIBaseManager::SetNewState(EAvatarCoreAIState NewState, bool ForceState)
 {
 
@@ -146,31 +154,33 @@ void UAIBaseManager::SetNewState(EAvatarCoreAIState NewState, bool ForceState)
     }
 }
 
-void UAIBaseManager::SendResponse(const FString& Response, EAvatarCoreAIPromptRole Role = EAvatarCoreAIPromptRole::User, bool NotifyDelay = false, bool TriggerResponse = true)
+void UAIBaseManager::SendResponse(FAIMessage Message, bool NotifyDelay, bool TriggerResponse)
 {
-    if (CurrentAIState != EAvatarCoreAIState::Ready) {
+    if (CurrentAIState != EAvatarCoreAIState::Ready && TriggerResponse) {
-        FPrompt tmpPrompt;
+        FAIMessage tmpPrompt;
-        tmpPrompt.Prompt = Response;
+        tmpPrompt.Message = Message.Message;
-        tmpPrompt.Role = Role;
+        tmpPrompt.Role = Message.Role;
         ResponseQueue.Enqueue(tmpPrompt);
         if(CurrentAIState == EAvatarCoreAIState::Disconnected)
             ActivateAI();    
         return;
     }
 
+    AddMessageToArray(Message);
+
     AnswerCache.Empty();
     ResponseID++;
-	LastRequest = Response;
+	LastRequest = Message.Message;
     if (CurrentAIState < EAvatarCoreAIState::Ready || CurrentAIState >= EAvatarCoreAIState::Error) {
 		BroadcastAIError("AI is not ready to send a response", EAvatarCoreAIError::NotInitialized);
 		ActivateAI();
 		return;
 	}
     UAIBaseManager::SetNewState(EAvatarCoreAIState::Processing);
-	BroadcastAILog(FString::Printf(TEXT("AI Manager sent question/response: %s"), *Response));
+	BroadcastAILog(FString::Printf(TEXT("AI Manager sent question/response: %s"), *Message.Message));
     if (NotifyDelay)
         UAIBaseManager::StartDelayedAnswerTimer();
-	SendResponseChild(Response, Role, NotifyDelay, TriggerResponse);
+	SendResponseChild(Message, NotifyDelay, TriggerResponse);
 }
 
 void UAIBaseManager::RepeatText(FString TextToRepeat, bool DoRephrase)
@@ -182,7 +192,10 @@ void UAIBaseManager::RepeatText(FString TextToRepeat, bool DoRephrase)
         Instruction = "[REPHRASE] " + TextToRepeat;
 	else
 		Instruction = "[REPEAT] " + TextToRepeat;
-	SendResponse(Instruction, EAvatarCoreAIPromptRole::System, false, true);
+    FAIMessage tmpPrompt;
+    tmpPrompt.Message = Instruction;
+    tmpPrompt.Role = EAvatarCoreAIPromptRole::System;
+	SendResponse(tmpPrompt, false, true);
 }
 
 void UAIBaseManager::ClearAI()
@@ -365,7 +378,10 @@ void UAIBaseManager::CommandFinished(const FString& Command, const FString& Payl
         BroadcastAILog(FString::Printf(TEXT("Command '%s' ran successfully. Answer: %s"), *Command, *Payload), true);
     else
         BroadcastAILog(FString::Printf(TEXT("Command '%s' ran successfully."), *Command), true);
-    SendResponse(Payload);
+    FAIMessage FinishedCommandMessage;
+    FinishedCommandMessage.Role = EAvatarCoreAIPromptRole::System;
+    FinishedCommandMessage.Message = Payload;
+    SendResponse(FinishedCommandMessage, false, true);
 }
 
 void UAIBaseManager::CommandFailed(const FString& Command, const FString& Payload)
@@ -373,7 +389,26 @@ void UAIBaseManager::CommandFailed(const FString& Command, const FString& Payloa
     functionCallRunning = false;
     SetNewState(EAvatarCoreAIState::Ready);
     BroadcastAILog(FString::Printf(TEXT("Command '%s' failed. Sending: %s"), *Command, *Payload), true);
-    SendResponse(Payload);
+    FAIMessage FailedCommandMessage;
+    FailedCommandMessage.Role = EAvatarCoreAIPromptRole::System;
+    FailedCommandMessage.Message = Payload;
+    SendResponse(FailedCommandMessage, false, true);
+}
+
+void UAIBaseManager::AddMessageToArray(FAIMessage NewMessage)
+{
+    PreviousMessages.Add(NewMessage);
+
+    // Remove oldest if over limit
+    if (CurrentConfig->MaxMessages > -1 && PreviousMessages.Num() > CurrentConfig->MaxMessages)
+    {
+        PreviousMessages.RemoveAt(0); // removes oldest (first element)
+    }
+}
+
+TArray<FAIMessage> UAIBaseManager::GetAllPreviousMessage()
+{
+    return PreviousMessages;
 }
 
 void UAIBaseManager::StartDelayedAnswerTimer()
@@ -409,6 +444,7 @@ void UAIBaseManager::OnAIResponse(const FString& Chunk, bool IsFinal)
 {
     ClearDelayedAnswerTimer();
        
+
     FString UpdatedAnswer;
     {
         FScopeLock Lock(&AnswerCacheCriticalSection);
@@ -433,6 +469,13 @@ void UAIBaseManager::OnAIResponse(const FString& Chunk, bool IsFinal)
                     BroadcastAILog(CopyAnswer, true);
         });
     }
+    if(IsFinal)
+    {
+        FAIMessage tmpAIAnswer;
+        tmpAIAnswer.Role = EAvatarCoreAIPromptRole::Assistant;
+        tmpAIAnswer.Message = Chunk;
+        AddMessageToArray(tmpAIAnswer);
+    }
 }
 
 void UAIBaseManager::AddSystemInstructions(const TArray<FSystemInstruction> SystemInstructions, bool WipeCurrent = true)

Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/RealtimeAPI/AvatarCoreAIRealtime.cpp

@@ -22,7 +22,6 @@ void UAvatarCoreAIRealtime::InitAIManagerChild(UAIBaseConfig* AIConfig, AActor*
 
 void UAvatarCoreAIRealtime::DeinitAIManager()
 {
-    //UAvatarCoreAIRealtime::DeactivateAI();
     UAIBaseManager::DeinitAIManager();
 }
 
@@ -34,13 +33,13 @@ void UAvatarCoreAIRealtime::ActivateAI()
 
 void UAvatarCoreAIRealtime::DeactivateAI()
 {
-    UAvatarCoreAIRealtime::DisconnectFromWebSocket();
     UAIBaseManager::DeactivateAI();
+    UAvatarCoreAIRealtime::DisconnectFromWebSocket();
 }
 
-void UAvatarCoreAIRealtime::SendResponseChild(const FString& Response, EAvatarCoreAIPromptRole Role, bool NotifyDelay = false, bool TriggerResponse = true)
+void UAvatarCoreAIRealtime::SendResponseChild(FAIMessage Message, bool NotifyDelay, bool TriggerResponse)
 {
-    UAvatarCoreAIRealtime::CreateConversationItem(Response, Role, TriggerResponse);
+    UAvatarCoreAIRealtime::CreateConversationItem(Message, TriggerResponse);
 }
 
 void UAvatarCoreAIRealtime::ClearAI()
@@ -218,6 +217,8 @@ void UAvatarCoreAIRealtime::ConnectToWebSocket()
 
 void UAvatarCoreAIRealtime::DisconnectFromWebSocket()
 {
+    if (!bIsWebsocketConnected)
+        return;
     bIsWebsocketConnected = false;
     bIsSessionInitialized = false;
     UAvatarCoreAIRealtime::SetNewState(EAvatarCoreAIState::Disconnected);
@@ -263,7 +264,7 @@ void UAvatarCoreAIRealtime::WebSocketSendType(const FString& type)
     }
 }
 
-void UAvatarCoreAIRealtime::CreateConversationItem(const FString& message, EAvatarCoreAIPromptRole role, bool triggerResponse)
+void UAvatarCoreAIRealtime::CreateConversationItem(FAIMessage Message, bool triggerResponse)
 {
     TSharedPtr<FJsonObject> RootObject = MakeShareable(new FJsonObject);
     RootObject->SetStringField("type", "conversation.item.create");
@@ -271,15 +272,17 @@ void UAvatarCoreAIRealtime::CreateConversationItem(const FString& message, EAvat
     // Create the item object
     TSharedPtr<FJsonObject> ItemObject = MakeShareable(new FJsonObject);
     ItemObject->SetStringField("type", "message");
-    ItemObject->SetStringField("role", UAvatarCoreAIRealtime::GetRoleAsString(role));
+    ItemObject->SetStringField("role", UAvatarCoreAIRealtime::GetRoleAsString(Message.Role));
 
     // Create the content array with an audio object inside
     TArray<TSharedPtr<FJsonValue>> ContentArray;
 
     TSharedPtr<FJsonObject> AudioObject = MakeShareable(new FJsonObject);
-    AudioObject->SetStringField("type", "input_text");
+    if(Message.Role == EAvatarCoreAIPromptRole::User || triggerResponse)
-    AudioObject->SetStringField("text", message);
+        AudioObject->SetStringField("type", "input_text");
-
+    else
+        AudioObject->SetStringField("type", "output_text"); 
+    AudioObject->SetStringField("text", Message.Message);
     ContentArray.Add(MakeShareable(new FJsonValueObject(AudioObject)));
     ItemObject->SetArrayField("content", ContentArray);
 
@@ -288,7 +291,8 @@ void UAvatarCoreAIRealtime::CreateConversationItem(const FString& message, EAvat
 
     // Convert the root JSON object to a string
     FString OutputString;
-    TSharedRef<TJsonWriter<>> Writer = TJsonWriterFactory<>::Create(&OutputString);
+    TSharedRef<TJsonWriter<TCHAR, TCondensedJsonPrintPolicy<TCHAR>>> Writer =
+        TJsonWriterFactory<TCHAR, TCondensedJsonPrintPolicy<TCHAR>>::Create(&OutputString);
     FJsonSerializer::Serialize(RootObject.ToSharedRef(), Writer);
 
     UAvatarCoreAIRealtime::WebSocketSend(OutputString); //Send the Message
@@ -323,7 +327,8 @@ void UAvatarCoreAIRealtime::CreateReseponse()
 
     // Convert the root JSON object to a string
     FString OutputString;
-    TSharedRef<TJsonWriter<>> Writer = TJsonWriterFactory<>::Create(&OutputString);
+    TSharedRef<TJsonWriter<TCHAR, TCondensedJsonPrintPolicy<TCHAR>>> Writer =
+        TJsonWriterFactory<TCHAR, TCondensedJsonPrintPolicy<TCHAR>>::Create(&OutputString);
     FJsonSerializer::Serialize(RootObject.ToSharedRef(), Writer);
 
     UAvatarCoreAIRealtime::WebSocketSend(OutputString);
@@ -527,7 +532,7 @@ void UAvatarCoreAIRealtime::OnWebSocketConnectionStringReceived(const FString& M
                                 if (CurrentRetries < MaxRetries)
                                 {
                                     CurrentRetries++;
-                                    BroadcastAILog(FString::Printf(TEXT("Response.done is way to fast. Something is fishy. Let's try again OpenAI!")), true);
+                                    BroadcastAILog(FString::Printf(TEXT("Response.done is way to fast. Something is fishy. Let's try again OpenAI! %s"), *TypeString), true);
                                     CreateReseponse();
                                 }
                                 else {

Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/AIBaseConfig.h

@@ -59,4 +59,8 @@ public:
 	// How many seconds before the delayedanswer event is triggered
 	UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Settings", meta = (ExposeOnSpawn = "true"))
 	float DelayAnswerSeconds = 1.5f;
+
+	// How many messages to keep in context (-1 for infinite)
+	UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Settings", meta = (ExposeOnSpawn = "true"))
+	int MaxMessages = -1;
 };

Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/AIBaseManager.h

@@ -16,6 +16,7 @@ DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FOnAILog, FString, Message);
 DECLARE_DYNAMIC_MULTICAST_DELEGATE_TwoParams(FOnAIError, FString, ErrorMessage, EAvatarCoreAIError, ErrorType);
 DECLARE_DYNAMIC_MULTICAST_DELEGATE_TwoParams(FMulticastDelegateRealtimeAPIAudioChunk, const TArray<uint8>, PCMData, bool, IsFinal);
 DECLARE_DYNAMIC_MULTICAST_DELEGATE(FOnAIDelayedAnswer);
+DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FOnAIActivationStateChanged, bool, IsActive);
 
 class UMCPManager;
 
@@ -50,6 +51,11 @@ public:
     //If the AI Model creates audio itself it can be forwarded the TTS Manager. Not a blueprint, because blueprint does not care about uint8
     FMulticastDelegateRealtimeAPIAudioChunk OnAudioChunk;
 
+    //AI activated and ready for action
+    UPROPERTY(BlueprintAssignable, Category = "AvatarCoreAI|Events")
+    FOnAIActivationStateChanged OnAIActivationStateChanged;
+
+
     /**
      * Initializes the AI Manager with the given config and adds this UObject to the root set.
      */
@@ -86,14 +92,14 @@ public:
     void OnAIReady();
 
     // Is the AI activated and ready?
-    UFUNCTION(BlueprintCallable, Category = "AvatarCoreAI")
+    UFUNCTION(BlueprintPure, Category = "AvatarCoreAI")
     bool IsAIActivated();
 
     /**
     * Deinitalize AI - when going to Idle for example
     */
     UFUNCTION(BlueprintCallable, Category = "AvatarCoreAI")
-    virtual void DeactivateAI() {};
+    virtual void DeactivateAI();
 
     /**
     * Change State of the AI Manager.
@@ -105,10 +111,10 @@ public:
     * Send Response/Question to the AI Model. If NotifyDelay is true call the DelayedAnswer Event when time defined in AIConfig has passed.
     */
     UFUNCTION(BlueprintCallable, Category = "AvatarCoreAI")
-    void SendResponse(const FString& Response, EAvatarCoreAIPromptRole Role, bool NotifyDelay, bool TriggerResponse);
+    void SendResponse(FAIMessage Message, bool NotifyDelay, bool TriggerResponse);
 
     UFUNCTION(BlueprintCallable, Category = "AvatarCoreAI")
-    virtual void SendResponseChild(const FString& Response, EAvatarCoreAIPromptRole Role, bool NotifyDelay, bool TriggerResponse) {};
+    virtual void SendResponseChild(FAIMessage Message, bool NotifyDelay, bool TriggerResponse) {};
 
     /**
     * Make the AI Model repeat the Text.
@@ -223,6 +229,12 @@ protected:
     UFUNCTION()
     void CommandFailed(const FString& Command, const FString& Payload);
 
+    //Add System/User/Assistant Message to memory archive
+    void AddMessageToArray(FAIMessage NewMessage);
+
+    //Add System/User/Assistant Message to memory archive
+    TArray<FAIMessage> GetAllPreviousMessage();
+
     //MCP Log Event
     UFUNCTION()
     void OnMCPLogReceived(const FString& LogMessage);
@@ -284,8 +296,11 @@ protected:
     //Handle Delayed Answer
     FTimerHandle DelayedAnswerTimer;
 
+    //All previous messages in a Queue
+    TArray<FAIMessage> PreviousMessages;
+
 private:
 
-	TQueue<FPrompt, EQueueMode::Spsc> ResponseQueue;
+	TQueue<FAIMessage, EQueueMode::Spsc> ResponseQueue;
 
 };

Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/AvatarCoreAIEnumsAndStructs.h

@@ -51,6 +51,18 @@ enum class EOpenAIAssistantRequestType : uint8
 	DeleteRun
 };
 
+USTRUCT(BlueprintType)
+struct FAIMessage
+{
+	GENERATED_BODY()
+
+	UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AI")
+	EAvatarCoreAIPromptRole Role = EAvatarCoreAIPromptRole::User;
+
+	UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AI")
+	FString Message;
+};
+
 USTRUCT(BlueprintType)
 struct FSystemInstruction
 {
@@ -88,15 +100,3 @@ struct FMCPToolInfo
 		OutputScheme = "";
 	}
 };
-
-USTRUCT(BlueprintType)
-struct FPrompt
-{
-	GENERATED_BODY()
-
-	UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AI")
-	FString Prompt;
-
-	UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AI")
-	EAvatarCoreAIPromptRole Role;
-};

Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/RealtimeAPI/AvatarCoreAIRealtime.h

@@ -149,7 +149,7 @@ public:
 	void ActivateAI() override;
 	void DeactivateAI() override;
 	void UpdateSession() override;
-	void SendResponseChild(const FString& Response, EAvatarCoreAIPromptRole Role, bool NotifyDelay, bool TriggerResponse) override;
+	void SendResponseChild(FAIMessage Message, bool NotifyDelay, bool TriggerResponse) override;
 	void ClearAI() override;
 
 	void ConnectToWebSocket();
@@ -162,7 +162,7 @@ public:
 	void WebSocketSendType(const FString& type);
 
 	UFUNCTION(BlueprintCallable, Category = "AvatarCore AI|RealtimeAPI")
-	void CreateConversationItem(const FString& message, EAvatarCoreAIPromptRole role, bool triggerResponse = true);
+	void CreateConversationItem(FAIMessage Message, bool triggerResponse = true);
 
 	UFUNCTION(BlueprintCallable, Category = "AvatarCore AI|RealtimeAPI")
 	void CreateReseponse();

Unreal/Plugins/AvatarCore_Manager/AvatarCore_Manager.uplugin

@@ -2,8 +2,8 @@
 	"FileVersion": 3,
 	"Version": 1,
 	"VersionName": "1.1",
-	"FriendlyName": "Avatar Core Manager",
+	"FriendlyName": "AvatarCore Manager",
-	"Description": "Avatar Core Manager",
+	"Description": "AvatarCore Manager",
 	"Category": "Other",
 	"CreatedBy": "b.ReX GmbH",
 	"CreatedByURL": "",

Unreal/Plugins/AvatarCore_MetaHuman/AvatarCore_MetaHuman.uplugin

@@ -2,7 +2,7 @@
 	"FileVersion": 3,
 	"Version": 1,
 	"VersionName": "1.0",
-	"FriendlyName": "AvatarCore_MetaHuman",
+	"FriendlyName": "AvatarCore MetaHuman",
 	"Description": "Everything neccessary for MetaHuman Avatars",
 	"Category": "Other",
 	"CreatedBy": "b.ReX GmbH",