diff --git a/Unreal/Plugins/RuntimeMetaHumanLipSync_5.6/Source/RuntimeMetaHumanLipSync/Private/RealisticMetaHumanLipSyncGenerator.cpp b/Unreal/Plugins/RuntimeMetaHumanLipSync_5.6/Source/RuntimeMetaHumanLipSync/Private/RealisticMetaHumanLipSyncGenerator.cpp index 371914d..10e5039 100644 --- a/Unreal/Plugins/RuntimeMetaHumanLipSync_5.6/Source/RuntimeMetaHumanLipSync/Private/RealisticMetaHumanLipSyncGenerator.cpp +++ b/Unreal/Plugins/RuntimeMetaHumanLipSync_5.6/Source/RuntimeMetaHumanLipSync/Private/RealisticMetaHumanLipSyncGenerator.cpp @@ -121,14 +121,8 @@ URealisticMetaHumanLipSyncGenerator* URealisticMetaHumanLipSyncGenerator::Create { URealisticMetaHumanLipSyncGenerator* Generator = NewObject(); #if PLATFORM_WINDOWS || PLATFORM_MAC || PLATFORM_LINUX || PLATFORM_IOS || PLATFORM_ANDROID - if (!Generator->OrtEnvironment) - { - Generator->OrtEnvironment = MakeShared(ORT_LOGGING_LEVEL_WARNING, "MetaHumanLipSyncModel"); - Generator->GeneratorConfig = Config; - - // Start async model loading - Generator->InitializeModelAsync(); - } + Generator->GeneratorConfig = Config; + Generator->InitializeModelAsync(); #endif return Generator; } @@ -137,18 +131,10 @@ URealisticMetaHumanLipSyncGenerator* URealisticMetaHumanLipSyncGenerator::Create { URealisticMetaHumanLipSyncGenerator* Generator = NewObject(); #if PLATFORM_WINDOWS || PLATFORM_MAC || PLATFORM_LINUX || PLATFORM_IOS || PLATFORM_ANDROID - if (!Generator->OrtEnvironment) - { - Generator->OrtEnvironment = MakeShared(ORT_LOGGING_LEVEL_WARNING, "MetaHumanLipSyncMoodModel"); - Generator->GeneratorConfigWithMood = Config; - - // Apply configuration settings - Generator->CurrentLookaheadMs = Config.LookaheadMs; - Generator->CurrentOutputType = Config.OutputType; - - // Start async model loading - Generator->InitializeModelAsync(); - } + Generator->GeneratorConfigWithMood = Config; + Generator->CurrentLookaheadMs = Config.LookaheadMs; + Generator->CurrentOutputType = Config.OutputType; + Generator->InitializeModelAsync(); #endif return Generator; } @@ -1167,29 +1153,7 @@ void URealisticMetaHumanLipSyncGenerator::InitializeModelAsync(TFunctionCacheMutex); - if (TSharedPtr* ExistingModel = ModelCache->CachedModels.Find(CurrentModelKey)) - { - if (ExistingModel->IsValid() && (*ExistingModel)->bIsValid) - { - (*ExistingModel)->ReferenceCount++; - bool bInitSuccess = InitializeOnnxSessionFromCache(*ExistingModel); - ModelLoadState = bInitSuccess ? ELipSyncModelLoadState::Loaded : ELipSyncModelLoadState::Failed; - - OnModelLoaded.Broadcast(bInitSuccess); - if (OnComplete) - { - OnComplete(bInitSuccess); - } - return; - } - } - } - - // Get the asset path for async loading + // Get the asset path for async loading (cheap, game-thread safe) const FRuntimeMetaHumanLipSyncModule* RuntimeMetaHumanLipSyncModulePtr = static_cast(FModuleManager::Get().GetModule("RuntimeMetaHumanLipSync")); if (!RuntimeMetaHumanLipSyncModulePtr) { @@ -1218,8 +1182,61 @@ void URealisticMetaHumanLipSyncGenerator::InitializeModelAsync(TFunction CachedHit; + if (ModelCache.IsValid()) + { + FScopeLock Lock(&ModelCache->CacheMutex); + if (TSharedPtr* ExistingModel = ModelCache->CachedModels.Find(CurrentModelKey)) + { + if (ExistingModel->IsValid() && (*ExistingModel)->bIsValid) + { + (*ExistingModel)->ReferenceCount++; + CachedHit = *ExistingModel; + } + } + } + TWeakObjectPtr WeakThis(this); + + if (CachedHit.IsValid()) + { + // Fix B: Cache hit — create Ort::Env + Ort::Session on a background thread, not the game thread + AsyncTask(ENamedThreads::AnyBackgroundThreadNormalTask, [WeakThis, CachedHit, OnComplete]() + { + URealisticMetaHumanLipSyncGenerator* Generator = WeakThis.Get(); + if (!Generator) + { + return; + } + +#if PLATFORM_WINDOWS || PLATFORM_MAC || PLATFORM_LINUX || PLATFORM_IOS || PLATFORM_ANDROID + const bool bIsMood = Generator->GeneratorConfigWithMood.IsSet(); + Generator->OrtEnvironment = MakeShared(ORT_LOGGING_LEVEL_WARNING, + bIsMood ? "MetaHumanLipSyncMoodModel" : "MetaHumanLipSyncModel"); +#endif + bool bInitSuccess = Generator->InitializeOnnxSessionFromCache(CachedHit); + + // Only marshal the lightweight state update back to the game thread + AsyncTask(ENamedThreads::GameThread, [WeakThis, bInitSuccess, OnComplete]() + { + URealisticMetaHumanLipSyncGenerator* Generator = WeakThis.Get(); + if (Generator) + { + Generator->ModelLoadState = bInitSuccess ? ELipSyncModelLoadState::Loaded : ELipSyncModelLoadState::Failed; + if (bInitSuccess) + { + UE_LOG(LogRuntimeMetaHumanLipSync, Log, TEXT("Successfully loaded lip sync model from cache asynchronously")); + } + Generator->OnModelLoaded.Broadcast(bInitSuccess); + if (OnComplete) OnComplete(bInitSuccess); + } + }); + }); + return; + } + + // Fix A + C: Cache miss — use StreamableManager for async asset loading, then create ORT objects on background thread UAssetManager::GetStreamableManager().RequestAsyncLoad( AssetPath, [WeakThis, OnComplete]() @@ -1230,7 +1247,7 @@ void URealisticMetaHumanLipSyncGenerator::InitializeModelAsync(TFunction CachedModel; + if (Generator->ModelCache.IsValid()) { - auto CachedModel = Generator->ModelCache->GetOrLoadModel(Generator->CurrentModelKey, + CachedModel = Generator->ModelCache->GetOrLoadModel(Generator->CurrentModelKey, [WeakThis](TSharedPtr& ModelData) -> bool { URealisticMetaHumanLipSyncGenerator* Generator = WeakThis.Get(); @@ -1253,35 +1270,36 @@ void URealisticMetaHumanLipSyncGenerator::InitializeModelAsync(TFunctionLoadModelDataFromLoadedAsset(ModelData); }); - - if (CachedModel.IsValid() && CachedModel->bIsValid) + } + + if (CachedModel.IsValid() && CachedModel->bIsValid) + { + // Fix A + C: Construct Ort::Env and Ort::Session here on the background thread +#if PLATFORM_WINDOWS || PLATFORM_MAC || PLATFORM_LINUX || PLATFORM_IOS || PLATFORM_ANDROID + const bool bIsMood = Generator->GeneratorConfigWithMood.IsSet(); + Generator->OrtEnvironment = MakeShared(ORT_LOGGING_LEVEL_WARNING, + bIsMood ? "MetaHumanLipSyncMoodModel" : "MetaHumanLipSyncModel"); +#endif + bool bInitSuccess = Generator->InitializeOnnxSessionFromCache(CachedModel); + + // Only marshal the lightweight state update back to the game thread + AsyncTask(ENamedThreads::GameThread, [WeakThis, bInitSuccess, OnComplete]() { - // Initialize ONNX session from cached data back on game thread - AsyncTask(ENamedThreads::GameThread, [WeakThis, CachedModel, OnComplete]() + URealisticMetaHumanLipSyncGenerator* Generator = WeakThis.Get(); + if (Generator) { - URealisticMetaHumanLipSyncGenerator* Generator = WeakThis.Get(); - if (Generator) + Generator->ModelLoadState = bInitSuccess ? ELipSyncModelLoadState::Loaded : ELipSyncModelLoadState::Failed; + if (bInitSuccess) { - bool bInitSuccess = Generator->InitializeOnnxSessionFromCache(CachedModel); - Generator->ModelLoadState = bInitSuccess ? ELipSyncModelLoadState::Loaded : ELipSyncModelLoadState::Failed; - - if (bInitSuccess) - { - UE_LOG(LogRuntimeMetaHumanLipSync, Log, TEXT("Successfully loaded lip sync model asynchronously")); - } - - Generator->OnModelLoaded.Broadcast(bInitSuccess); - - if (OnComplete) - { - OnComplete(bInitSuccess); - } + UE_LOG(LogRuntimeMetaHumanLipSync, Log, TEXT("Successfully loaded lip sync model asynchronously")); } - }); - return; - } + Generator->OnModelLoaded.Broadcast(bInitSuccess); + if (OnComplete) OnComplete(bInitSuccess); + } + }); + return; } - + // Failed to load AsyncTask(ENamedThreads::GameThread, [WeakThis, OnComplete]() { @@ -1290,11 +1308,7 @@ void URealisticMetaHumanLipSyncGenerator::InitializeModelAsync(TFunctionModelLoadState = ELipSyncModelLoadState::Failed; Generator->OnModelLoaded.Broadcast(false); - - if (OnComplete) - { - OnComplete(false); - } + if (OnComplete) OnComplete(false); } }); });