Matched with AvatarBase

Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/AIBaseManager.cpp

@@ -115,6 +115,17 @@ void UAIBaseManager::DeinitAIManager()
     BroadcastAILog(FString::Printf(TEXT("AI Manager deinitialized!")), false);
 }
 
+void UAIBaseManager::OnAIReady()
+{
+    UAIBaseManager::SetNewState(EAvatarCoreAIState::Ready);
+    if (!ResponseQueue.IsEmpty())
+    {
+        FString QueuedPrompt;
+        ResponseQueue.Dequeue(QueuedPrompt);
+        UAIBaseManager::SendResponse(QueuedPrompt, false, true);
+    }
+}
+
 void UAIBaseManager::SetNewState(EAvatarCoreAIState NewState, bool ForceState)
 {
 
@@ -126,6 +137,13 @@ void UAIBaseManager::SetNewState(EAvatarCoreAIState NewState, bool ForceState)
 
 void UAIBaseManager::SendResponse(const FString& Response, bool NotifyDelay = false, bool TriggerResponse = true)
 {
+    if (CurrentAIState != EAvatarCoreAIState::Ready) {
+        ResponseQueue.Enqueue(Response);
+        if(CurrentAIState == EAvatarCoreAIState::Disconnected)
+            ActivateAI();    
+        return;
+    }
+
     AnswerCache.Empty();
     ResponseID++;
 	LastRequest = Response;
@@ -134,10 +152,11 @@ void UAIBaseManager::SendResponse(const FString& Response, bool NotifyDelay = fa
 		ActivateAI();
 		return;
 	}
+    UAIBaseManager::SetNewState(EAvatarCoreAIState::Processing);
 	BroadcastAILog(FString::Printf(TEXT("AI Manager sent question/response: %s"), *Response));
     if (NotifyDelay)
         UAIBaseManager::StartDelayedAnswerTimer();
-	//Override in Child
+	SendResponseChild(Response, NotifyDelay, TriggerResponse);
 }
 
 void UAIBaseManager::RepeatText(FString TextToRepeat, bool DoRephrase)
@@ -154,6 +173,8 @@ void UAIBaseManager::RepeatText(FString TextToRepeat, bool DoRephrase)
 
 void UAIBaseManager::ClearAI()
 {
+    ResponseQueue.Empty();
+
     if (CurrentAIState <= EAvatarCoreAIState::Ready)
         return;
     UAIBaseManager::SetNewState(EAvatarCoreAIState::Ready, true);
@@ -309,6 +330,7 @@ void UAIBaseManager::ClearMCPCommand()
 
 void UAIBaseManager::CommandFinished(const FString& Command, const FString& Payload)
 {
+    UE_LOG(LogTemp, Warning, TEXT("Processing"));
     SetNewState(EAvatarCoreAIState::Processing);
     functionCallRunning = false;
     if (bDebugMode)

Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/RealtimeAPI/AvatarCoreAIRealtime.cpp

@@ -28,6 +28,7 @@ void UAvatarCoreAIRealtime::DeinitAIManager()
 
 void UAvatarCoreAIRealtime::ActivateAI()
 {
+    UAIBaseManager::SetNewState(EAvatarCoreAIState::Initializing);
     UAvatarCoreAIRealtime::ConnectToWebSocket();
 }
 
@@ -37,9 +38,8 @@ void UAvatarCoreAIRealtime::DeactivateAI()
     UAIBaseManager::DeactivateAI();
 }
 
-void UAvatarCoreAIRealtime::SendResponse(const FString& Response, bool NotifyDelay = false, bool TriggerResponse = true)
+void UAvatarCoreAIRealtime::SendResponseChild(const FString& Response, bool NotifyDelay = false, bool TriggerResponse = true)
 {
-    UAIBaseManager::SendResponse(Response, NotifyDelay, TriggerResponse);
     UAvatarCoreAIRealtime::CreateConversationItem(Response, EOpenAIRoleType::User, TriggerResponse);
 }
 
@@ -219,7 +219,7 @@ void UAvatarCoreAIRealtime::ConnectToWebSocket()
 void UAvatarCoreAIRealtime::DisconnectFromWebSocket()
 {
     bIsWebsocketConnected = false;
-
+    bIsSessionInitialized = false;
     UAvatarCoreAIRealtime::SetNewState(EAvatarCoreAIState::Disconnected);
     // Clean up all accumulated audio data
     RequestAudioData.Empty();
@@ -295,7 +295,6 @@ void UAvatarCoreAIRealtime::CreateConversationItem(const FString& message, EOpen
 
     if (triggerResponse) {
         UAvatarCoreAIRealtime::CreateReseponse();
-        UAvatarCoreAIRealtime::SetNewState(EAvatarCoreAIState::Processing);
     }
 }
 
@@ -361,9 +360,11 @@ void UAvatarCoreAIRealtime::OnWebSocketConnectionError(const FString& Error)
 
 void UAvatarCoreAIRealtime::OnWebSocketConnectionConnected()
 {
-    BroadcastAILog("OpenAI Websocket connected");
+    if(!bIsWebsocketConnected)
-    bIsWebsocketConnected = true;
+    {
-    UAvatarCoreAIRealtime::UpdateSession();
+        BroadcastAILog("OpenAI Websocket connected");
+        bIsWebsocketConnected = true;
+    }  
 }
 
 void UAvatarCoreAIRealtime::OnWebSocketConnectionStringReceived(const FString& Message)
@@ -423,10 +424,16 @@ void UAvatarCoreAIRealtime::OnWebSocketConnectionStringReceived(const FString& M
                     }
                 }
             }
-            else if (TypeString == "session.updated") {
+            else if (TypeString == "session.created")
-                UAvatarCoreAIRealtime::SetNewState(EAvatarCoreAIState::Ready);
+            {
+                UAvatarCoreAIRealtime::UpdateSession();
+                return;
+            }
+            else if (TypeString == "session.updated")
+            {
+                UAIBaseManager::OnAIReady();
+                return;
             }
-
             // Match the request type
             if (TypeParts[0] == "session")
                 RequestType = EOpenAIRequestType::session;

Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/AIBaseManager.h

@@ -81,6 +81,10 @@ public:
     UFUNCTION(BlueprintCallable, Category = "AvatarCoreAI")
     virtual void UpdateSession() {};
 
+    // They AI is ready to answer
+    UFUNCTION()
+    void OnAIReady();
+
     /**
     * Deinitalize AI - when going to Idle for example
     */
@@ -97,7 +101,10 @@ public:
     * Send Response/Question to the AI Model. If NotifyDelay is true call the DelayedAnswer Event when time defined in AIConfig has passed.
     */
     UFUNCTION(BlueprintCallable, Category = "AvatarCoreAI")
-    virtual void SendResponse(const FString& Response, bool NotifyDelay, bool TriggerResponse);
+    void SendResponse(const FString& Response, bool NotifyDelay, bool TriggerResponse);
+
+    UFUNCTION(BlueprintCallable, Category = "AvatarCoreAI")
+    virtual void SendResponseChild(const FString& Response, bool NotifyDelay, bool TriggerResponse) {};
 
     /**
     * Make the AI Model repeat the Text.
@@ -264,6 +271,8 @@ protected:
     //Handle Delayed Answer
     FTimerHandle DelayedAnswerTimer;
 
-protected:
+private:
+
+	TQueue<FString, EQueueMode::Spsc> ResponseQueue;
 
 };

Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/RealtimeAPI/AvatarCoreAIRealtime.h

@@ -156,7 +156,7 @@ public:
 	void ActivateAI() override;
 	void DeactivateAI() override;
 	void UpdateSession() override;
-	void SendResponse(const FString& Response, bool NotifyDelay, bool TriggerResponse) override;
+	void SendResponseChild(const FString& Response, bool NotifyDelay, bool TriggerResponse) override;
 	void ClearAI() override;
 
 	void ConnectToWebSocket();
@@ -203,6 +203,10 @@ private:
 	bool ResponseAudioDone = false;
 	bool ResponseTextDone = false;
 
+	bool bIsSessionInitialized = false;
+	bool bWaitingForSessionUpdatedAck = false;
+	FString PendingSessionUpdateEventId;
+
 	// Map to store accumulated audio data for each request
 	TMap<FString, TArray<uint8>> RequestAudioData;