diff --git a/Unreal/Content/Schema/Spie_Config.schema.json b/Unreal/Content/Schema/Spie_Config.schema.json index 31a217e..0872a20 100644 --- a/Unreal/Content/Schema/Spie_Config.schema.json +++ b/Unreal/Content/Schema/Spie_Config.schema.json @@ -335,7 +335,8 @@ ], "STTSpecialWords": [ "SPIE" - ] + ], + "PTTPostRollTime": 0.40000000596046448 }, "category": "STT Settings", "fields": @@ -450,6 +451,10 @@ { "type": "array", "itemsType": "string" + }, + "PTTPostRollTime": + { + "type": "float" } } } diff --git a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/MCP/FastMCP/FastMCPManager.cpp b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/MCP/FastMCP/FastMCPManager.cpp index a5e7c27..06eb1d2 100644 --- a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/MCP/FastMCP/FastMCPManager.cpp +++ b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Private/MCP/FastMCP/FastMCPManager.cpp @@ -496,7 +496,7 @@ void UFastMCPManager::OnCommandExecuted(FHttpRequestPtr Request, FHttpResponsePt const TSharedPtr* ResultObject; if (JsonObject->TryGetObjectField(TEXT("result"), ResultObject)) { - if ((*ResultObject)->HasField(TEXT("isError")) && (*ResultObject)->GetBoolField("isError")) + if ((*ResultObject)->HasField(TEXT("isError")) && (*ResultObject)->GetBoolField(TEXT("isError"))) { LogError(FString::Printf(TEXT("Command execution failed: %s - %s"), *CurrentExecutingCommand, *ResponseContent), EMCPManagerError::ToolError); OnMCPCommandFailed.Broadcast(CurrentExecutingCommand, ResponseContent); diff --git a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/AIBaseManager.h b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/AIBaseManager.h index fd2372c..6345ea1 100644 --- a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/AIBaseManager.h +++ b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/AIBaseManager.h @@ -26,19 +26,23 @@ class AVATARCORE_AI_API UAIBaseManager : public UObject public: - // BlueprintAssignable Events + // AI subtitle received (chunk-delta and complete answer) UPROPERTY(BlueprintAssignable, Category = "AvatarCoreAI|Events") FOnAISubtitle OnAISubtitle; + //AI module has changed state UPROPERTY(BlueprintAssignable, Category = "AvatarCoreAI|Events") FOnAIStateChanged OnAIStateChanged; + //AI module logging UPROPERTY(BlueprintAssignable, Category = "AvatarCoreAI|Events") FOnAILog OnAILog; + //AI module error UPROPERTY(BlueprintAssignable, Category = "AvatarCoreAI|Events") FOnAIError OnAIError; + //It took a very long time to start answering - we can use this to playback a filler speech UPROPERTY(BlueprintAssignable, Category = "AvatarCoreAI|Events") FOnAIDelayedAnswer OnAIDelayedAnswer; @@ -70,6 +74,12 @@ public: UFUNCTION(BlueprintCallable, Category = "AvatarCoreAI") virtual void ActivateAI() {}; + /** + * Update Session in case something has changed with the prompt or the functions to call + */ + UFUNCTION(BlueprintCallable, Category = "AvatarCoreAI") + virtual void UpdateSession() {}; + /** * Deinitalize AI - when going to Idle for example */ @@ -118,15 +128,27 @@ public: UFUNCTION(BlueprintCallable, Category = "AvatarCoreAI|Events") virtual void OnAIResponse(const FString& Chunk, bool IsFinal); + /** + * Add a new system instruction by name. + */ UFUNCTION(BlueprintCallable, Category = "AvatarCoreAI") void AddSystemInstruction(const FName Name, const FString NewSystemInstruction); + /** + * Remove a system instruction by name. + */ UFUNCTION(BlueprintCallable, Category = "AvatarCoreAI") void RemoveSystemInstruction(const FName Name); + /** + * Clear all System Instruction. + */ UFUNCTION(BlueprintCallable, Category = "AvatarCoreAI") void ClearAllSystemInstructios(); + /** + * Timeout Handling + */ void ResetRequestTimeout(); void ClearRequestTimeout(); UFUNCTION() @@ -144,9 +166,9 @@ public: UFUNCTION(BlueprintPure, Category = "AvatarCoreAI|Commands") TArray GetAvailableCommands(); - /** - * Runs a command by name with the given payload, rebinding completion/failure events. - */ + /** + * Runs a command by name with the given payload, rebinding completion/failure events. + */ void RunMCPCommand(FString CommandName, FString Payload); // Clear all running commands @@ -166,22 +188,23 @@ protected: UFUNCTION() void CommandFailed(const FString& Command, const FString& Payload); - // MCP Event Handlers + //MCP Log Event UFUNCTION() void OnMCPLogReceived(const FString& LogMessage); + //MCP Error Event UFUNCTION() void OnMCPErrorReceived(const FString& ErrorMessage, EMCPManagerError ErrorType); + //MCP state changed UFUNCTION() void OnMCPStateChanged(EMCPManagerState NewState); + //Tracking if it takes too long until answer chunks are coming in UFUNCTION() void StartDelayedAnswerTimer(); - UFUNCTION() void ClearDelayedAnswerTimer(); - UFUNCTION() void OnDelayedAnswer(); diff --git a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/MCP/FastMCP/FastMCPConfig.h b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/MCP/FastMCP/FastMCPConfig.h index 9c49067..5bbac45 100644 --- a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/MCP/FastMCP/FastMCPConfig.h +++ b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/MCP/FastMCP/FastMCPConfig.h @@ -16,12 +16,15 @@ class AVATARCORE_AI_API UFastMCPConfig : public UMCPBaseConfig public: + //Direction to the Script that start FastMCP UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|MCP", meta = (ExposeOnSpawn = "true")) FString MCPExecutable = FPaths::ProjectPluginsDir() +"AvatarCore_AI/Source/ThirdParty/MCPServer/FastMCP/FastMCPServer.bat"; + //Custom python environment - "python" will use the system default UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreTTS|CoquiTTS", meta = (ExposeOnSpawn = "true")) FString PythonPath = "python"; + //Additional arguments to pass UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|MCP", meta = (ExposeOnSpawn = "true")) FString Arguments = ""; diff --git a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/RealtimeAPI/AvatarCoreAIRealtime.h b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/RealtimeAPI/AvatarCoreAIRealtime.h index 6f83d31..e8ca00e 100644 --- a/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/RealtimeAPI/AvatarCoreAIRealtime.h +++ b/Unreal/Plugins/AvatarCore_AI/Source/AvatarCore_AI/Public/RealtimeAPI/AvatarCoreAIRealtime.h @@ -155,13 +155,10 @@ public: void DeinitAIManager() override; void ActivateAI() override; void DeactivateAI() override; + void UpdateSession() override; void SendResponse(const FString& Response, bool NotifyDelay = false, FString OverrideInstruction = "") override; void ClearAI() override; - //RealtimeAPI only functions - UFUNCTION(BlueprintCallable, Category = "AvatarCore AI|RealtimeAPI") - void UpdateSession(); - void ConnectToWebSocket(); void DisconnectFromWebSocket(); diff --git a/Unreal/Plugins/AvatarCore_AI/Source/ThirdParty/MCPServer/FastMCP/lancedb/documents.lance/_transactions/0-7032d51b-cf75-439d-b546-8a49bd9f7c9c.txn b/Unreal/Plugins/AvatarCore_AI/Source/ThirdParty/MCPServer/FastMCP/lancedb/documents.lance/_transactions/0-7032d51b-cf75-439d-b546-8a49bd9f7c9c.txn deleted file mode 100644 index e513fa8..0000000 Binary files a/Unreal/Plugins/AvatarCore_AI/Source/ThirdParty/MCPServer/FastMCP/lancedb/documents.lance/_transactions/0-7032d51b-cf75-439d-b546-8a49bd9f7c9c.txn and /dev/null differ diff --git a/Unreal/Plugins/AvatarCore_AI/Source/ThirdParty/MCPServer/FastMCP/lancedb/documents.lance/_transactions/1-e4aac62e-df93-4f1b-a546-43a8516937ae.txn b/Unreal/Plugins/AvatarCore_AI/Source/ThirdParty/MCPServer/FastMCP/lancedb/documents.lance/_transactions/1-e4aac62e-df93-4f1b-a546-43a8516937ae.txn deleted file mode 100644 index 83e34b8..0000000 Binary files a/Unreal/Plugins/AvatarCore_AI/Source/ThirdParty/MCPServer/FastMCP/lancedb/documents.lance/_transactions/1-e4aac62e-df93-4f1b-a546-43a8516937ae.txn and /dev/null differ diff --git a/Unreal/Plugins/AvatarCore_AI/Source/ThirdParty/MCPServer/FastMCP/lancedb/documents.lance/_versions/1.manifest b/Unreal/Plugins/AvatarCore_AI/Source/ThirdParty/MCPServer/FastMCP/lancedb/documents.lance/_versions/1.manifest deleted file mode 100644 index d779827..0000000 Binary files a/Unreal/Plugins/AvatarCore_AI/Source/ThirdParty/MCPServer/FastMCP/lancedb/documents.lance/_versions/1.manifest and /dev/null differ diff --git a/Unreal/Plugins/AvatarCore_AI/Source/ThirdParty/MCPServer/FastMCP/lancedb/documents.lance/_versions/2.manifest b/Unreal/Plugins/AvatarCore_AI/Source/ThirdParty/MCPServer/FastMCP/lancedb/documents.lance/_versions/2.manifest deleted file mode 100644 index 8ad68c7..0000000 Binary files a/Unreal/Plugins/AvatarCore_AI/Source/ThirdParty/MCPServer/FastMCP/lancedb/documents.lance/_versions/2.manifest and /dev/null differ diff --git a/Unreal/Plugins/AvatarCore_AI/Source/ThirdParty/MCPServer/FastMCP/lancedb/documents.lance/data/3e53aab5-bc5c-4db7-9f2c-31cdd3423b52.lance b/Unreal/Plugins/AvatarCore_AI/Source/ThirdParty/MCPServer/FastMCP/lancedb/documents.lance/data/3e53aab5-bc5c-4db7-9f2c-31cdd3423b52.lance deleted file mode 100644 index cd06d9e..0000000 Binary files a/Unreal/Plugins/AvatarCore_AI/Source/ThirdParty/MCPServer/FastMCP/lancedb/documents.lance/data/3e53aab5-bc5c-4db7-9f2c-31cdd3423b52.lance and /dev/null differ diff --git a/Unreal/Plugins/AvatarCore_Manager/Content/UnrealCommands/AICommandTestBigHead.uasset b/Unreal/Plugins/AvatarCore_Manager/Content/UnrealCommands/AICommandTestBigHead.uasset new file mode 100644 index 0000000..8a65d1b Binary files /dev/null and b/Unreal/Plugins/AvatarCore_Manager/Content/UnrealCommands/AICommandTestBigHead.uasset differ diff --git a/Unreal/Plugins/AvatarCore_MetaHuman/Content/Animation/AnimationConfigs/AnimationConfig_Base.uasset b/Unreal/Plugins/AvatarCore_MetaHuman/Content/Animation/AnimationConfigs/AnimationConfig_Base.uasset index 5b0b2f7..77ac7e9 100644 Binary files a/Unreal/Plugins/AvatarCore_MetaHuman/Content/Animation/AnimationConfigs/AnimationConfig_Base.uasset and b/Unreal/Plugins/AvatarCore_MetaHuman/Content/Animation/AnimationConfigs/AnimationConfig_Base.uasset differ diff --git a/Unreal/Plugins/AvatarCore_MetaHuman/Content/BP/Interface/BI_AvatarAnimBP.uasset b/Unreal/Plugins/AvatarCore_MetaHuman/Content/BP/Interface/BI_AvatarAnimBP.uasset index f78d3b5..d28bd4e 100644 Binary files a/Unreal/Plugins/AvatarCore_MetaHuman/Content/BP/Interface/BI_AvatarAnimBP.uasset and b/Unreal/Plugins/AvatarCore_MetaHuman/Content/BP/Interface/BI_AvatarAnimBP.uasset differ diff --git a/Unreal/Plugins/AvatarCore_MetaHuman/Content/BP/Interface/BI_RetargetedAvatarAnimBP.uasset b/Unreal/Plugins/AvatarCore_MetaHuman/Content/BP/Interface/BI_RetargetedAvatarAnimBP.uasset index 52c991f..5fd2999 100644 Binary files a/Unreal/Plugins/AvatarCore_MetaHuman/Content/BP/Interface/BI_RetargetedAvatarAnimBP.uasset and b/Unreal/Plugins/AvatarCore_MetaHuman/Content/BP/Interface/BI_RetargetedAvatarAnimBP.uasset differ diff --git a/Unreal/Plugins/AvatarCore_MetaHuman/Content/BP/MetaHuman/AvatarBase.uasset b/Unreal/Plugins/AvatarCore_MetaHuman/Content/BP/MetaHuman/AvatarBase.uasset index 7f98be8..35d4322 100644 Binary files a/Unreal/Plugins/AvatarCore_MetaHuman/Content/BP/MetaHuman/AvatarBase.uasset and b/Unreal/Plugins/AvatarCore_MetaHuman/Content/BP/MetaHuman/AvatarBase.uasset differ diff --git a/Unreal/Plugins/AvatarCore_MetaHuman/Content/BP/MetaHuman/AvatarCore_AnimInst_Body.uasset b/Unreal/Plugins/AvatarCore_MetaHuman/Content/BP/MetaHuman/AvatarCore_AnimInst_Body.uasset index b11d931..259be73 100644 Binary files a/Unreal/Plugins/AvatarCore_MetaHuman/Content/BP/MetaHuman/AvatarCore_AnimInst_Body.uasset and b/Unreal/Plugins/AvatarCore_MetaHuman/Content/BP/MetaHuman/AvatarCore_AnimInst_Body.uasset differ diff --git a/Unreal/Plugins/AvatarCore_MetaHuman/Content/BP/MetaHuman/AvatarCore_AnimInst_BodyRetarget.uasset b/Unreal/Plugins/AvatarCore_MetaHuman/Content/BP/MetaHuman/AvatarCore_AnimInst_BodyRetarget.uasset index d00575c..a1b4b51 100644 Binary files a/Unreal/Plugins/AvatarCore_MetaHuman/Content/BP/MetaHuman/AvatarCore_AnimInst_BodyRetarget.uasset and b/Unreal/Plugins/AvatarCore_MetaHuman/Content/BP/MetaHuman/AvatarCore_AnimInst_BodyRetarget.uasset differ diff --git a/Unreal/Plugins/AvatarCore_MetaHuman/Content/BP/MetaHuman/AvatarCore_AnimInst_Face.uasset b/Unreal/Plugins/AvatarCore_MetaHuman/Content/BP/MetaHuman/AvatarCore_AnimInst_Face.uasset index 48ac1b3..18ce1c7 100644 Binary files a/Unreal/Plugins/AvatarCore_MetaHuman/Content/BP/MetaHuman/AvatarCore_AnimInst_Face.uasset and b/Unreal/Plugins/AvatarCore_MetaHuman/Content/BP/MetaHuman/AvatarCore_AnimInst_Face.uasset differ diff --git a/Unreal/Plugins/AvatarCore_MetaHuman/Content/Materials/MI_GrayTexture_Head.uasset b/Unreal/Plugins/AvatarCore_MetaHuman/Content/Materials/MI_GrayTexture_Head.uasset new file mode 100644 index 0000000..6422a6d Binary files /dev/null and b/Unreal/Plugins/AvatarCore_MetaHuman/Content/Materials/MI_GrayTexture_Head.uasset differ diff --git a/Unreal/Plugins/AvatarCore_MetaHuman/Content/Materials/M_GrayTexture_Head.uasset b/Unreal/Plugins/AvatarCore_MetaHuman/Content/Materials/M_GrayTexture_Head.uasset new file mode 100644 index 0000000..aa6c6fd Binary files /dev/null and b/Unreal/Plugins/AvatarCore_MetaHuman/Content/Materials/M_GrayTexture_Head.uasset differ diff --git a/Unreal/Plugins/AvatarCore_MetaHuman/Content/Materials/Textures/T_Gray_Head_D.uasset b/Unreal/Plugins/AvatarCore_MetaHuman/Content/Materials/Textures/T_Gray_Head_D.uasset new file mode 100644 index 0000000..845ee06 Binary files /dev/null and b/Unreal/Plugins/AvatarCore_MetaHuman/Content/Materials/Textures/T_Gray_Head_D.uasset differ diff --git a/Unreal/Plugins/AvatarCore_MetaHuman/Content/Meshes/m_med_nrw_withHead.uasset b/Unreal/Plugins/AvatarCore_MetaHuman/Content/Meshes/m_med_nrw_withHead.uasset new file mode 100644 index 0000000..5d81791 Binary files /dev/null and b/Unreal/Plugins/AvatarCore_MetaHuman/Content/Meshes/m_med_nrw_withHead.uasset differ diff --git a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/STTManagerBase.cpp b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/STTManagerBase.cpp index 50a1c19..752daaf 100644 --- a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/STTManagerBase.cpp +++ b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Private/STTManagerBase.cpp @@ -257,7 +257,7 @@ void USTTManagerBase::PTTStateChanged(bool BtnPressed) PTTPostRollTimer, // handle to cancel timer at a later time this, // the owning object &USTTManagerBase::PTTRelease, // function to call on elapsed - PTTPostRollTime, // float delay until elapsed + ProcessorConfig->BaseSettings.PTTPostRollTime, // float delay until elapsed false); // looping? } else diff --git a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/STTStructs.h b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/STTStructs.h index 7bf75a2..6c21132 100644 --- a/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/STTStructs.h +++ b/Unreal/Plugins/AvatarCore_STT/Source/AvatarCore_STT/Public/STTStructs.h @@ -125,5 +125,7 @@ struct FSTTBaseSettings TArray STTReplacements; UPROPERTY(EditAnywhere, BlueprintReadWrite, meta = (ToolTip = "Special words that the transcription service needs to know (e.g. b.ReX or Bruce-B).", Category = "STT|Base")) TArray STTSpecialWords; + UPROPERTY(EditAnywhere, BlueprintReadWrite, meta = (ToolTip = "How long to keep transcribing when user released button. Users like to finish early ;)", Category = "STT|Base")) + float PTTPostRollTime = 0.4; }; \ No newline at end of file diff --git a/Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Private/TTSManagerBase.cpp b/Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Private/TTSManagerBase.cpp index 164966b..9072a1a 100644 --- a/Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Private/TTSManagerBase.cpp +++ b/Unreal/Plugins/AvatarCore_TTS/Source/AvatarCore_TTS/Private/TTSManagerBase.cpp @@ -117,7 +117,7 @@ void UTTSManagerBase::PlaybackSoundWave(USoundWave* InSoundWave) int32 NumSamplesToRemove = SampleRate * NumChannels * 0.1f; if (PCMData.Num() >= NumSamplesToRemove) { - PCMData.RemoveAt(0, NumSamplesToRemove, /* bAllowShrinking = */ false); + PCMData.RemoveAt(0, NumSamplesToRemove, EAllowShrinking::No); } AddAudioChunk(PCMData, true); }