Made model load Async (thanks claude)

Unreal/Plugins/RuntimeMetaHumanLipSync_5.6/Source/RuntimeMetaHumanLipSync/Private/RealisticMetaHumanLipSyncGenerator.cpp

@@ -121,14 +121,8 @@ URealisticMetaHumanLipSyncGenerator* URealisticMetaHumanLipSyncGenerator::Create
 {
 	URealisticMetaHumanLipSyncGenerator* Generator = NewObject<URealisticMetaHumanLipSyncGenerator>();
 #if PLATFORM_WINDOWS || PLATFORM_MAC || PLATFORM_LINUX || PLATFORM_IOS || PLATFORM_ANDROID
-	if (!Generator->OrtEnvironment)
+	Generator->GeneratorConfig = Config;
-	{
+	Generator->InitializeModelAsync();
-		Generator->OrtEnvironment = MakeShared<Ort::Env>(ORT_LOGGING_LEVEL_WARNING, "MetaHumanLipSyncModel");
-		Generator->GeneratorConfig = Config;
-        
-		// Start async model loading
-		Generator->InitializeModelAsync();
-	}
 #endif
 	return Generator;
 }
@@ -137,18 +131,10 @@ URealisticMetaHumanLipSyncGenerator* URealisticMetaHumanLipSyncGenerator::Create
 {
 	URealisticMetaHumanLipSyncGenerator* Generator = NewObject<URealisticMetaHumanLipSyncGenerator>();
 #if PLATFORM_WINDOWS || PLATFORM_MAC || PLATFORM_LINUX || PLATFORM_IOS || PLATFORM_ANDROID
-	if (!Generator->OrtEnvironment)
+	Generator->GeneratorConfigWithMood = Config;
-	{
+	Generator->CurrentLookaheadMs = Config.LookaheadMs;
-		Generator->OrtEnvironment = MakeShared<Ort::Env>(ORT_LOGGING_LEVEL_WARNING, "MetaHumanLipSyncMoodModel");
+	Generator->CurrentOutputType = Config.OutputType;
-		Generator->GeneratorConfigWithMood = Config;
+	Generator->InitializeModelAsync();
-        
-		// Apply configuration settings
-		Generator->CurrentLookaheadMs = Config.LookaheadMs;
-		Generator->CurrentOutputType = Config.OutputType;
-        
-		// Start async model loading
-		Generator->InitializeModelAsync();
-	}
 #endif
 	return Generator;
 }
@@ -1167,29 +1153,7 @@ void URealisticMetaHumanLipSyncGenerator::InitializeModelAsync(TFunction<void(bo
     ModelLoadState = ELipSyncModelLoadState::Loading;
     CurrentModelKey = GetModelCacheKey();
     
-    // Check if model is already cached first
+    // Get the asset path for async loading (cheap, game-thread safe)
-    if (ModelCache.IsValid())
-    {
-        FScopeLock Lock(&ModelCache->CacheMutex);
-        if (TSharedPtr<FLipSyncModelCache::FCachedModelData>* ExistingModel = ModelCache->CachedModels.Find(CurrentModelKey))
-        {
-            if (ExistingModel->IsValid() && (*ExistingModel)->bIsValid)
-            {
-                (*ExistingModel)->ReferenceCount++;
-                bool bInitSuccess = InitializeOnnxSessionFromCache(*ExistingModel);
-                ModelLoadState = bInitSuccess ? ELipSyncModelLoadState::Loaded : ELipSyncModelLoadState::Failed;
-                
-                OnModelLoaded.Broadcast(bInitSuccess);
-                if (OnComplete)
-                {
-                    OnComplete(bInitSuccess);
-                }
-                return;
-            }
-        }
-    }
-    
-    // Get the asset path for async loading
     const FRuntimeMetaHumanLipSyncModule* RuntimeMetaHumanLipSyncModulePtr = static_cast<FRuntimeMetaHumanLipSyncModule*>(FModuleManager::Get().GetModule("RuntimeMetaHumanLipSync"));
     if (!RuntimeMetaHumanLipSyncModulePtr)
     {
@@ -1218,8 +1182,61 @@ void URealisticMetaHumanLipSyncGenerator::InitializeModelAsync(TFunction<void(bo
         return;
     }
 
-    // Use StreamableManager for async asset loading
+    // Fix B: Check cache hit — grab a reference under the lock, then release the lock before going async
+    TSharedPtr<FLipSyncModelCache::FCachedModelData> CachedHit;
+    if (ModelCache.IsValid())
+    {
+        FScopeLock Lock(&ModelCache->CacheMutex);
+        if (TSharedPtr<FLipSyncModelCache::FCachedModelData>* ExistingModel = ModelCache->CachedModels.Find(CurrentModelKey))
+        {
+            if (ExistingModel->IsValid() && (*ExistingModel)->bIsValid)
+            {
+                (*ExistingModel)->ReferenceCount++;
+                CachedHit = *ExistingModel;
+            }
+        }
+    }
+
     TWeakObjectPtr<URealisticMetaHumanLipSyncGenerator> WeakThis(this);
+
+    if (CachedHit.IsValid())
+    {
+        // Fix B: Cache hit — create Ort::Env + Ort::Session on a background thread, not the game thread
+        AsyncTask(ENamedThreads::AnyBackgroundThreadNormalTask, [WeakThis, CachedHit, OnComplete]()
+        {
+            URealisticMetaHumanLipSyncGenerator* Generator = WeakThis.Get();
+            if (!Generator)
+            {
+                return;
+            }
+
+#if PLATFORM_WINDOWS || PLATFORM_MAC || PLATFORM_LINUX || PLATFORM_IOS || PLATFORM_ANDROID
+            const bool bIsMood = Generator->GeneratorConfigWithMood.IsSet();
+            Generator->OrtEnvironment = MakeShared<Ort::Env>(ORT_LOGGING_LEVEL_WARNING,
+                bIsMood ? "MetaHumanLipSyncMoodModel" : "MetaHumanLipSyncModel");
+#endif
+            bool bInitSuccess = Generator->InitializeOnnxSessionFromCache(CachedHit);
+
+            // Only marshal the lightweight state update back to the game thread
+            AsyncTask(ENamedThreads::GameThread, [WeakThis, bInitSuccess, OnComplete]()
+            {
+                URealisticMetaHumanLipSyncGenerator* Generator = WeakThis.Get();
+                if (Generator)
+                {
+                    Generator->ModelLoadState = bInitSuccess ? ELipSyncModelLoadState::Loaded : ELipSyncModelLoadState::Failed;
+                    if (bInitSuccess)
+                    {
+                        UE_LOG(LogRuntimeMetaHumanLipSync, Log, TEXT("Successfully loaded lip sync model from cache asynchronously"));
+                    }
+                    Generator->OnModelLoaded.Broadcast(bInitSuccess);
+                    if (OnComplete) OnComplete(bInitSuccess);
+                }
+            });
+        });
+        return;
+    }
+
+    // Fix A + C: Cache miss — use StreamableManager for async asset loading, then create ORT objects on background thread
     UAssetManager::GetStreamableManager().RequestAsyncLoad(
         AssetPath,
         [WeakThis, OnComplete]()
@@ -1230,7 +1247,7 @@ void URealisticMetaHumanLipSyncGenerator::InitializeModelAsync(TFunction<void(bo
                 return;
             }
 
-            // Asset is now loaded, extract model data on background thread
+            // Asset is now loaded — extract model data and create ORT session entirely on background thread
             AsyncTask(ENamedThreads::AnyBackgroundThreadNormalTask, [WeakThis, OnComplete]()
             {
                 URealisticMetaHumanLipSyncGenerator* Generator = WeakThis.Get();
@@ -1239,11 +1256,11 @@ void URealisticMetaHumanLipSyncGenerator::InitializeModelAsync(TFunction<void(bo
                     return;
                 }
 
-                bool bSuccess = false;
+                TSharedPtr<FLipSyncModelCache::FCachedModelData> CachedModel;
-                
+
                 if (Generator->ModelCache.IsValid())
                 {
-                    auto CachedModel = Generator->ModelCache->GetOrLoadModel(Generator->CurrentModelKey, 
+                    CachedModel = Generator->ModelCache->GetOrLoadModel(Generator->CurrentModelKey,
                         [WeakThis](TSharedPtr<FLipSyncModelCache::FCachedModelData>& ModelData) -> bool
                         {
                             URealisticMetaHumanLipSyncGenerator* Generator = WeakThis.Get();
@@ -1253,35 +1270,36 @@ void URealisticMetaHumanLipSyncGenerator::InitializeModelAsync(TFunction<void(bo
                             }
                             return Generator->LoadModelDataFromLoadedAsset(ModelData);
                         });
-                    
+                }
-                    if (CachedModel.IsValid() && CachedModel->bIsValid)
+
+                if (CachedModel.IsValid() && CachedModel->bIsValid)
+                {
+                    // Fix A + C: Construct Ort::Env and Ort::Session here on the background thread
+#if PLATFORM_WINDOWS || PLATFORM_MAC || PLATFORM_LINUX || PLATFORM_IOS || PLATFORM_ANDROID
+                    const bool bIsMood = Generator->GeneratorConfigWithMood.IsSet();
+                    Generator->OrtEnvironment = MakeShared<Ort::Env>(ORT_LOGGING_LEVEL_WARNING,
+                        bIsMood ? "MetaHumanLipSyncMoodModel" : "MetaHumanLipSyncModel");
+#endif
+                    bool bInitSuccess = Generator->InitializeOnnxSessionFromCache(CachedModel);
+
+                    // Only marshal the lightweight state update back to the game thread
+                    AsyncTask(ENamedThreads::GameThread, [WeakThis, bInitSuccess, OnComplete]()
                     {
-                        // Initialize ONNX session from cached data back on game thread
+                        URealisticMetaHumanLipSyncGenerator* Generator = WeakThis.Get();
-                        AsyncTask(ENamedThreads::GameThread, [WeakThis, CachedModel, OnComplete]()
+                        if (Generator)
                         {
-                            URealisticMetaHumanLipSyncGenerator* Generator = WeakThis.Get();
+                            Generator->ModelLoadState = bInitSuccess ? ELipSyncModelLoadState::Loaded : ELipSyncModelLoadState::Failed;
-                            if (Generator)
+                            if (bInitSuccess)
                             {
-                                bool bInitSuccess = Generator->InitializeOnnxSessionFromCache(CachedModel);
+                                UE_LOG(LogRuntimeMetaHumanLipSync, Log, TEXT("Successfully loaded lip sync model asynchronously"));
-                                Generator->ModelLoadState = bInitSuccess ? ELipSyncModelLoadState::Loaded : ELipSyncModelLoadState::Failed;
-                                
-                                if (bInitSuccess)
-                                {
-                                    UE_LOG(LogRuntimeMetaHumanLipSync, Log, TEXT("Successfully loaded lip sync model asynchronously"));
-                                }
-                                
-                                Generator->OnModelLoaded.Broadcast(bInitSuccess);
-                                
-                                if (OnComplete)
-                                {
-                                    OnComplete(bInitSuccess);
-                                }
                             }
-                        });
+                            Generator->OnModelLoaded.Broadcast(bInitSuccess);
-                        return;
+                            if (OnComplete) OnComplete(bInitSuccess);
-                    }
+                        }
+                    });
+                    return;
                 }
-                
+
                 // Failed to load
                 AsyncTask(ENamedThreads::GameThread, [WeakThis, OnComplete]()
                 {
@@ -1290,11 +1308,7 @@ void URealisticMetaHumanLipSyncGenerator::InitializeModelAsync(TFunction<void(bo
                     {
                         Generator->ModelLoadState = ELipSyncModelLoadState::Failed;
                         Generator->OnModelLoaded.Broadcast(false);
-                        
+                        if (OnComplete) OnComplete(false);
-                        if (OnComplete)
-                        {
-                            OnComplete(false);
-                        }
                     }
                 });
             });