diff --git a/Unreal/Content/Project/BP/Avatars/PersonalityConfigs/DA_PersonalityConfig_Ben_BREX.uasset b/Unreal/Content/Project/BP/Avatars/PersonalityConfigs/DA_PersonalityConfig_Ben_BREX.uasset index a4aa6a5..a6c793f 100644 --- a/Unreal/Content/Project/BP/Avatars/PersonalityConfigs/DA_PersonalityConfig_Ben_BREX.uasset +++ b/Unreal/Content/Project/BP/Avatars/PersonalityConfigs/DA_PersonalityConfig_Ben_BREX.uasset @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:61c322940011d34bd41803946eb71b20cf9f9c5461c0f22d4dab3d8b1b881452 -size 2460 +oid sha256:cadff3de50ef2b52625501590859655582163bd4c59704aad6953407c13a828c +size 3252 diff --git a/Unreal/Content/Project/BP/Avatars/PersonalityConfigs/DA_PersonalityConfig_Jasmin.uasset b/Unreal/Content/Project/BP/Avatars/PersonalityConfigs/DA_PersonalityConfig_Jasmin.uasset index 4c5b80b..d7dcc1d 100644 --- a/Unreal/Content/Project/BP/Avatars/PersonalityConfigs/DA_PersonalityConfig_Jasmin.uasset +++ b/Unreal/Content/Project/BP/Avatars/PersonalityConfigs/DA_PersonalityConfig_Jasmin.uasset @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:0a3bbf50f71fce2d1ea7f96c17ca3121312ad749924638b25ef09e60e384d69e -size 2342 +oid sha256:64220e4fd3bd950b0ffb8603e987aeec00464f32f06b82a761eca652c7a904aa +size 3251 diff --git a/Unreal/Content/Project/BP/BP_Project_Manager.uasset b/Unreal/Content/Project/BP/BP_Project_Manager.uasset index 5be9bdd..be47970 100644 --- a/Unreal/Content/Project/BP/BP_Project_Manager.uasset +++ b/Unreal/Content/Project/BP/BP_Project_Manager.uasset @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:83a04b88087fb7952b52768098cb9882202d840eb33c6f6699b7e8d8258e1678 -size 2878104 +oid sha256:87f5522be35f56eb92f12e1439b605ab1c9e510c11d609e905fc271edc6d6fd5 +size 2879432 diff --git a/Unreal/Content/Project/BP/Base/PC_Project.uasset b/Unreal/Content/Project/BP/Base/PC_Project.uasset index 99bc58e..154bda5 100644 --- a/Unreal/Content/Project/BP/Base/PC_Project.uasset +++ b/Unreal/Content/Project/BP/Base/PC_Project.uasset @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:5442b3132d40d8b428cc089641258e0b9216bf3ab6e241076b47ab53e3dee48e -size 192278 +oid sha256:13bec056f665ba472344937919315ce1665496e11ee6fd378df350dac28441a0 +size 199807 diff --git a/Unreal/Content/Project/Widgets/W_Main.uasset b/Unreal/Content/Project/Widgets/W_Main.uasset index 0ce58f6..7401370 100644 --- a/Unreal/Content/Project/Widgets/W_Main.uasset +++ b/Unreal/Content/Project/Widgets/W_Main.uasset @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:b6e30c23a1e32dd9a157eb5465887cc9d17b560c35ad2cfd86296eba41ea1375 -size 402522 +oid sha256:54a4717639a69f9ddce98ed7243777823ca794f45a0475360f796b1b21ff2394 +size 407923 diff --git a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/AIBaseManager.cpp b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/AIBaseManager.cpp index bc50cc2..4e2cff4 100644 --- a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/AIBaseManager.cpp +++ b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/AIBaseManager.cpp @@ -126,9 +126,9 @@ void UAIBaseManager::OnAIReady() UAIBaseManager::SetNewState(EAvatarCoreAIState::Ready); if (!ResponseQueue.IsEmpty()) { - FString QueuedPrompt; + FPrompt QueuedPrompt; ResponseQueue.Dequeue(QueuedPrompt); - UAIBaseManager::SendResponse(QueuedPrompt, false, true); + UAIBaseManager::SendResponse(QueuedPrompt.Prompt, QueuedPrompt.Role, false, true); } } @@ -146,10 +146,13 @@ void UAIBaseManager::SetNewState(EAvatarCoreAIState NewState, bool ForceState) } } -void UAIBaseManager::SendResponse(const FString& Response, bool NotifyDelay = false, bool TriggerResponse = true) +void UAIBaseManager::SendResponse(const FString& Response, EAvatarCoreAIPromptRole Role = EAvatarCoreAIPromptRole::User, bool NotifyDelay = false, bool TriggerResponse = true) { if (CurrentAIState != EAvatarCoreAIState::Ready) { - ResponseQueue.Enqueue(Response); + FPrompt tmpPrompt; + tmpPrompt.Prompt = Response; + tmpPrompt.Role = Role; + ResponseQueue.Enqueue(tmpPrompt); if(CurrentAIState == EAvatarCoreAIState::Disconnected) ActivateAI(); return; @@ -167,7 +170,7 @@ void UAIBaseManager::SendResponse(const FString& Response, bool NotifyDelay = fa BroadcastAILog(FString::Printf(TEXT("AI Manager sent question/response: %s"), *Response)); if (NotifyDelay) UAIBaseManager::StartDelayedAnswerTimer(); - SendResponseChild(Response, NotifyDelay, TriggerResponse); + SendResponseChild(Response, Role, NotifyDelay, TriggerResponse); } void UAIBaseManager::RepeatText(FString TextToRepeat, bool DoRephrase) @@ -179,7 +182,7 @@ void UAIBaseManager::RepeatText(FString TextToRepeat, bool DoRephrase) Instruction = "[REPHRASE] " + TextToRepeat; else Instruction = "[REPEAT] " + TextToRepeat; - SendResponse(Instruction, false, true); + SendResponse(Instruction, EAvatarCoreAIPromptRole::System, false, true); } void UAIBaseManager::ClearAI() @@ -341,6 +344,19 @@ void UAIBaseManager::ClearMCPCommand() } } +FString UAIBaseManager::GetRoleAsString(EAvatarCoreAIPromptRole Role) +{ + switch (Role) { + case EAvatarCoreAIPromptRole::Assistant: + return "assistant"; + case EAvatarCoreAIPromptRole::System: + return "system"; + case EAvatarCoreAIPromptRole::User: + default: + return "user"; + } +} + void UAIBaseManager::CommandFinished(const FString& Command, const FString& Payload) { SetNewState(EAvatarCoreAIState::Ready); diff --git a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/RealtimeAPI/AvatarCoreAIRealtime.cpp b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/RealtimeAPI/AvatarCoreAIRealtime.cpp index c531433..2eafdb9 100644 --- a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/RealtimeAPI/AvatarCoreAIRealtime.cpp +++ b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/RealtimeAPI/AvatarCoreAIRealtime.cpp @@ -38,9 +38,9 @@ void UAvatarCoreAIRealtime::DeactivateAI() UAIBaseManager::DeactivateAI(); } -void UAvatarCoreAIRealtime::SendResponseChild(const FString& Response, bool NotifyDelay = false, bool TriggerResponse = true) +void UAvatarCoreAIRealtime::SendResponseChild(const FString& Response, EAvatarCoreAIPromptRole Role, bool NotifyDelay = false, bool TriggerResponse = true) { - UAvatarCoreAIRealtime::CreateConversationItem(Response, EOpenAIRoleType::User, TriggerResponse); + UAvatarCoreAIRealtime::CreateConversationItem(Response, Role, TriggerResponse); } void UAvatarCoreAIRealtime::ClearAI() @@ -263,7 +263,7 @@ void UAvatarCoreAIRealtime::WebSocketSendType(const FString& type) } } -void UAvatarCoreAIRealtime::CreateConversationItem(const FString& message, EOpenAIRoleType role, bool triggerResponse) +void UAvatarCoreAIRealtime::CreateConversationItem(const FString& message, EAvatarCoreAIPromptRole role, bool triggerResponse) { TSharedPtr RootObject = MakeShareable(new FJsonObject); RootObject->SetStringField("type", "conversation.item.create"); @@ -695,17 +695,3 @@ void UAvatarCoreAIRealtime::DebugSimulateConnectionDrop() SetNewState(EAvatarCoreAIState::Error, true); BroadcastAIError("Simulated socket error 3008", EAvatarCoreAIError::ConnectionDropped); } - - -FString UAvatarCoreAIRealtime::GetRoleAsString(EOpenAIRoleType role) -{ - switch (role) { - case EOpenAIRoleType::Assistant: - return "assistant"; - case EOpenAIRoleType::System: - return "system"; - case EOpenAIRoleType::User: - default: - return "user"; - } -} diff --git a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/AIBaseManager.h b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/AIBaseManager.h index 3fd51ca..a2d2fd8 100644 --- a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/AIBaseManager.h +++ b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/AIBaseManager.h @@ -105,10 +105,10 @@ public: * Send Response/Question to the AI Model. If NotifyDelay is true call the DelayedAnswer Event when time defined in AIConfig has passed. */ UFUNCTION(BlueprintCallable, Category = "AvatarCoreAI") - void SendResponse(const FString& Response, bool NotifyDelay, bool TriggerResponse); + void SendResponse(const FString& Response, EAvatarCoreAIPromptRole Role, bool NotifyDelay, bool TriggerResponse); UFUNCTION(BlueprintCallable, Category = "AvatarCoreAI") - virtual void SendResponseChild(const FString& Response, bool NotifyDelay, bool TriggerResponse) {}; + virtual void SendResponseChild(const FString& Response, EAvatarCoreAIPromptRole Role, bool NotifyDelay, bool TriggerResponse) {}; /** * Make the AI Model repeat the Text. @@ -207,6 +207,9 @@ public: UFUNCTION(BlueprintCallable, Category = "AvatarCoreAI|MCP Commands") void ClearMCPCommand(); + UFUNCTION(BlueprintCallable, Category = "AvatarCoreAI|Helper") + static FString GetRoleAsString(EAvatarCoreAIPromptRole Role); + protected: /** @@ -283,6 +286,6 @@ protected: private: - TQueue ResponseQueue; + TQueue ResponseQueue; }; \ No newline at end of file diff --git a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/AvatarCoreAIEnumsAndStructs.h b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/AvatarCoreAIEnumsAndStructs.h index d27a4cf..42e9f03 100644 --- a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/AvatarCoreAIEnumsAndStructs.h +++ b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/AvatarCoreAIEnumsAndStructs.h @@ -18,6 +18,15 @@ enum class EAvatarCoreAIError : uint8 Unknown }; +UENUM(BlueprintType) +enum class EAvatarCoreAIPromptRole : uint8 +{ + User UMETA(DisplayName = "User"), + Assistant UMETA(DisplayName = "Assistant"), + System UMETA(DisplayName = "System"), +}; + + UENUM(BlueprintType) enum class EAvatarCoreAIState : uint8 { @@ -79,3 +88,15 @@ struct FMCPToolInfo OutputScheme = ""; } }; + +USTRUCT(BlueprintType) +struct FPrompt +{ + GENERATED_BODY() + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AI") + FString Prompt; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AI") + EAvatarCoreAIPromptRole Role; +}; diff --git a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/RealtimeAPI/AvatarCoreAIRealtime.h b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/RealtimeAPI/AvatarCoreAIRealtime.h index 0f28552..b5e0cc1 100644 --- a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/RealtimeAPI/AvatarCoreAIRealtime.h +++ b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/RealtimeAPI/AvatarCoreAIRealtime.h @@ -13,13 +13,6 @@ #include "RealtimeAPI/RealtimeAPIConfig.h" #include "AvatarCoreAIRealtime.generated.h" -UENUM(BlueprintType) -enum class EOpenAIRoleType : uint8 { - User UMETA(DisplayName = "User"), - Assistant UMETA(DisplayName = "Assistant"), - System UMETA(DisplayName = "System"), -}; - UENUM(BlueprintType) enum class EOpenAIRequestType : uint8 { undefined UMETA(DisplayName = "UNDEFINED"), @@ -156,7 +149,7 @@ public: void ActivateAI() override; void DeactivateAI() override; void UpdateSession() override; - void SendResponseChild(const FString& Response, bool NotifyDelay, bool TriggerResponse) override; + void SendResponseChild(const FString& Response, EAvatarCoreAIPromptRole Role, bool NotifyDelay, bool TriggerResponse) override; void ClearAI() override; void ConnectToWebSocket(); @@ -169,7 +162,7 @@ public: void WebSocketSendType(const FString& type); UFUNCTION(BlueprintCallable, Category = "AvatarCore AI|RealtimeAPI") - void CreateConversationItem(const FString& message, EOpenAIRoleType role, bool triggerResponse = true); + void CreateConversationItem(const FString& message, EAvatarCoreAIPromptRole role, bool triggerResponse = true); UFUNCTION(BlueprintCallable, Category = "AvatarCore AI|RealtimeAPI") void CreateReseponse(); @@ -198,8 +191,6 @@ public: private: URealtimeAPIConfig* RealtimeConfig; - - FString GetRoleAsString(EOpenAIRoleType role); FString CurrentRequestID; int32 CurrentRetries = 0; int32 MaxRetries = 2; diff --git a/Unreal/Plugins/AvatarCore_Manager/Content/AvatarCoreManager.uasset b/Unreal/Plugins/AvatarCore_Manager/Content/AvatarCoreManager.uasset index 283ec41..f62a565 100644 --- a/Unreal/Plugins/AvatarCore_Manager/Content/AvatarCoreManager.uasset +++ b/Unreal/Plugins/AvatarCore_Manager/Content/AvatarCoreManager.uasset @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:26cf0e6a390429735077a70d7ff385eb786433148afdaf86efe8f4a4b684430e -size 1851602 +oid sha256:e7de8da12b4cd144246eb0c80327740a5f4455729080d29dbc9ffa7190b9ddfb +size 1867172 diff --git a/Unreal/Plugins/AvatarCore_Manager/Content/StateManagement/BP_StateManager.uasset b/Unreal/Plugins/AvatarCore_Manager/Content/StateManagement/BP_StateManager.uasset index ffa3469..9f0ad8c 100644 --- a/Unreal/Plugins/AvatarCore_Manager/Content/StateManagement/BP_StateManager.uasset +++ b/Unreal/Plugins/AvatarCore_Manager/Content/StateManagement/BP_StateManager.uasset @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:2c7273fc694c27d8692d525808ab907d113e9f3344a857e894d75e751fac3a96 -size 710899 +oid sha256:93be3bfd54db98201a9ed5ee2059a5e58e0aa81ff08fdb55801b6e77fa6167f9 +size 712599 diff --git a/Unreal/Plugins/AvatarCore_Manager/Content/Widgets/Debug/Pages/W_DebugAvatarCoreSTT.uasset b/Unreal/Plugins/AvatarCore_Manager/Content/Widgets/Debug/Pages/W_DebugAvatarCoreSTT.uasset index 63e7f86..7f5768e 100644 --- a/Unreal/Plugins/AvatarCore_Manager/Content/Widgets/Debug/Pages/W_DebugAvatarCoreSTT.uasset +++ b/Unreal/Plugins/AvatarCore_Manager/Content/Widgets/Debug/Pages/W_DebugAvatarCoreSTT.uasset @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:c8f3913d82c3b1ac685284db14d28f0ca38b61631d2a12f66aede0b25b0a4cc2 -size 415726 +oid sha256:a003ec47034fc18212be19ba614cd62895b94e53cb9a53776a686457ce290d7c +size 402976 diff --git a/Unreal/Plugins/AvatarCore_MetaHuman/Content/BP/MetaHuman/BaseAvatar.uasset b/Unreal/Plugins/AvatarCore_MetaHuman/Content/BP/MetaHuman/BaseAvatar.uasset index b8ccbd7..bda107d 100644 --- a/Unreal/Plugins/AvatarCore_MetaHuman/Content/BP/MetaHuman/BaseAvatar.uasset +++ b/Unreal/Plugins/AvatarCore_MetaHuman/Content/BP/MetaHuman/BaseAvatar.uasset @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:d5d130eb9940b3ad8c6758820db27baf7ecced5abb09fee2636a608148ed4014 -size 2426536 +oid sha256:415f710dbd8046ed2373dcd2c8e6ff0cab64b6872197e205aaaa108345ef5493 +size 2422188 diff --git a/Unreal/Plugins/BTools/Content/ConfigPawn/AC_ConfigComponent.uasset b/Unreal/Plugins/BTools/Content/ConfigPawn/AC_ConfigComponent.uasset index 5d7ec3d..731d30a 100644 --- a/Unreal/Plugins/BTools/Content/ConfigPawn/AC_ConfigComponent.uasset +++ b/Unreal/Plugins/BTools/Content/ConfigPawn/AC_ConfigComponent.uasset @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:7034fc5e352f8ef65a50530640317b83b0d4910ca59342761926c6c8733dd4e7 -size 278956 +oid sha256:9e8defe613270af1410c81945b108c874cee93882151d91d379ddc79496e5256 +size 286875 diff --git a/Unreal/Plugins/BTools/Content/ConfigPawn/ConfigPawn.uasset b/Unreal/Plugins/BTools/Content/ConfigPawn/ConfigPawn.uasset index 4a1b0d9..b69a59b 100644 --- a/Unreal/Plugins/BTools/Content/ConfigPawn/ConfigPawn.uasset +++ b/Unreal/Plugins/BTools/Content/ConfigPawn/ConfigPawn.uasset @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:6d249f33d5690bc82f00a4469b67360b4866cbdbdc8123d22d8319dba95ccfaf -size 694809 +oid sha256:ac32f14fca5c87cb7fef93b13344794134e945572d6be41290f43dd50ab5b2dc +size 760964 diff --git a/Unreal/Plugins/BTools/Content/ConfigPawn/Structs/E_ConfigWidgetConfigType.uasset b/Unreal/Plugins/BTools/Content/ConfigPawn/Structs/E_ConfigWidgetConfigType.uasset new file mode 100644 index 0000000..cda2cf1 --- /dev/null +++ b/Unreal/Plugins/BTools/Content/ConfigPawn/Structs/E_ConfigWidgetConfigType.uasset @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8b91f14b3916dc27fc4c8bdfbf46744a80d3bee36c2eb9a792d65a6d2e92a6ea +size 4662 diff --git a/Unreal/Plugins/BTools/Content/ConfigPawn/Widgets/W_DebugConfigModeWidget.uasset b/Unreal/Plugins/BTools/Content/ConfigPawn/Widgets/W_DebugConfigModeWidget.uasset index 97e930e..4e60002 100644 --- a/Unreal/Plugins/BTools/Content/ConfigPawn/Widgets/W_DebugConfigModeWidget.uasset +++ b/Unreal/Plugins/BTools/Content/ConfigPawn/Widgets/W_DebugConfigModeWidget.uasset @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:4af5f40326c7ad3c17d46e0b7a75d2d421b0c21431be1c8fa7317cc66f3f92c1 -size 363570 +oid sha256:0bce1e5b863ed3cf547297d06802d262990d45ef71a1491401c939b9a2b704d4 +size 547108