209 changed files with 2320 additions and 197 deletions
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@ -0,0 +1,178 @@ |
|||||
|
// Fill out your copyright notice in the Description page of Project Settings.
|
||||
|
|
||||
|
|
||||
|
#include "Tools/OpenAiModerator.h" |
||||
|
|
||||
|
#include "HttpModule.h" |
||||
|
#include "Interfaces/IHttpResponse.h" |
||||
|
#include "Dom/JsonObject.h" |
||||
|
#include "Serialization/JsonReader.h" |
||||
|
#include "Serialization/JsonSerializer.h" |
||||
|
|
||||
|
FString UOpenAiModerator::StoredApiKey; |
||||
|
FString UOpenAiModerator::StoredModel = TEXT("omni-moderation-latest"); |
||||
|
TArray<TWeakObjectPtr<UOpenAiModerator>> UOpenAiModerator::ActiveRequests; |
||||
|
|
||||
|
void UOpenAiModerator::InitModeration(const FString& ApiKey, const FString& Model) |
||||
|
{ |
||||
|
StoredApiKey = ApiKey; |
||||
|
StoredModel = Model; |
||||
|
} |
||||
|
|
||||
|
UOpenAiModerator* UOpenAiModerator::UseModeration(UObject* WorldContextObject, const FString& Text) |
||||
|
{ |
||||
|
UOpenAiModerator* Action = NewObject<UOpenAiModerator>(); |
||||
|
Action->InputText = Text; |
||||
|
Action->RegisterWithGameInstance(WorldContextObject); |
||||
|
return Action; |
||||
|
} |
||||
|
|
||||
|
void UOpenAiModerator::Activate() |
||||
|
{ |
||||
|
if (StoredApiKey.IsEmpty()) |
||||
|
{ |
||||
|
UE_LOG(LogTemp, Error, TEXT("OpenAiModerator: API key not set. Call InitModeration first.")); |
||||
|
FModerationResult EmptyResult; |
||||
|
Passed.Broadcast(EmptyResult); |
||||
|
return; |
||||
|
} |
||||
|
|
||||
|
RequestStartTime = FPlatformTime::Seconds(); |
||||
|
ActiveRequests.Add(this); |
||||
|
|
||||
|
TSharedRef<IHttpRequest, ESPMode::ThreadSafe> HttpRequest = FHttpModule::Get().CreateRequest(); |
||||
|
HttpRequest->SetURL(TEXT("https://api.openai.com/v1/moderations")); |
||||
|
HttpRequest->SetVerb(TEXT("POST")); |
||||
|
HttpRequest->SetHeader(TEXT("Content-Type"), TEXT("application/json")); |
||||
|
HttpRequest->SetHeader(TEXT("Authorization"), FString::Printf(TEXT("Bearer %s"), *StoredApiKey)); |
||||
|
|
||||
|
const FString Body = FString::Printf(TEXT("{\"model\":\"%s\",\"input\":\"%s\"}"), |
||||
|
*StoredModel, |
||||
|
*InputText.ReplaceCharWithEscapedChar()); |
||||
|
|
||||
|
HttpRequest->SetContentAsString(Body); |
||||
|
|
||||
|
HttpRequest->OnProcessRequestComplete().BindUObject(this, &UOpenAiModerator::HandleHttpResponse); |
||||
|
|
||||
|
if (!HttpRequest->ProcessRequest()) |
||||
|
{ |
||||
|
UE_LOG(LogTemp, Error, TEXT("OpenAiModerator: Failed to start HTTP request.")); |
||||
|
FModerationResult EmptyResult; |
||||
|
Passed.Broadcast(EmptyResult); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
void UOpenAiModerator::ClearModeration() |
||||
|
{ |
||||
|
TArray<TWeakObjectPtr<UOpenAiModerator>> RequestsCopy = ActiveRequests; |
||||
|
ActiveRequests.Empty(); |
||||
|
|
||||
|
for (const TWeakObjectPtr<UOpenAiModerator>& Weak : RequestsCopy) |
||||
|
{ |
||||
|
if (UOpenAiModerator* Action = Weak.Get()) |
||||
|
{ |
||||
|
Action->SetReadyToDestroy(); |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
void UOpenAiModerator::HandleHttpResponse(FHttpRequestPtr Request, FHttpResponsePtr Response, bool bWasSuccessful) |
||||
|
{ |
||||
|
ActiveRequests.RemoveAll([this](const TWeakObjectPtr<UOpenAiModerator>& Weak) { return !Weak.IsValid() || Weak.Get() == this; }); |
||||
|
|
||||
|
const float ElapsedTime = static_cast<float>(FPlatformTime::Seconds() - RequestStartTime); |
||||
|
|
||||
|
FModerationResult Result; |
||||
|
Result.ExecutionTime = ElapsedTime; |
||||
|
|
||||
|
if (!bWasSuccessful || !Response.IsValid()) |
||||
|
{ |
||||
|
UE_LOG(LogTemp, Error, TEXT("OpenAiModerator: HTTP request failed.")); |
||||
|
Passed.Broadcast(Result); |
||||
|
return; |
||||
|
} |
||||
|
|
||||
|
const int32 Code = Response->GetResponseCode(); |
||||
|
const FString Body = Response->GetContentAsString(); |
||||
|
|
||||
|
if (Code < 200 || Code >= 300) |
||||
|
{ |
||||
|
UE_LOG(LogTemp, Error, TEXT("OpenAiModerator: HTTP %d — %s"), Code, *Body); |
||||
|
Passed.Broadcast(Result); |
||||
|
return; |
||||
|
} |
||||
|
|
||||
|
TSharedPtr<FJsonObject> RootObject; |
||||
|
TSharedRef<TJsonReader<>> Reader = TJsonReaderFactory<>::Create(Body); |
||||
|
if (!FJsonSerializer::Deserialize(Reader, RootObject) || !RootObject.IsValid()) |
||||
|
{ |
||||
|
UE_LOG(LogTemp, Error, TEXT("OpenAiModerator: Failed to parse response JSON.")); |
||||
|
Passed.Broadcast(Result); |
||||
|
return; |
||||
|
} |
||||
|
|
||||
|
const TArray<TSharedPtr<FJsonValue>>* ResultsArray; |
||||
|
if (!RootObject->TryGetArrayField(TEXT("results"), ResultsArray) || ResultsArray->Num() == 0) |
||||
|
{ |
||||
|
UE_LOG(LogTemp, Error, TEXT("OpenAiModerator: No results in response.")); |
||||
|
Passed.Broadcast(Result); |
||||
|
return; |
||||
|
} |
||||
|
|
||||
|
const TSharedPtr<FJsonObject>& FirstResult = (*ResultsArray)[0]->AsObject(); |
||||
|
if (!FirstResult.IsValid()) |
||||
|
{ |
||||
|
UE_LOG(LogTemp, Error, TEXT("OpenAiModerator: Invalid result object.")); |
||||
|
Passed.Broadcast(Result); |
||||
|
return; |
||||
|
} |
||||
|
|
||||
|
Result.bFlagged = FirstResult->GetBoolField(TEXT("flagged")); |
||||
|
|
||||
|
const TSharedPtr<FJsonObject>* CategoriesObject; |
||||
|
if (FirstResult->TryGetObjectField(TEXT("categories"), CategoriesObject)) |
||||
|
{ |
||||
|
FModerationCategories& C = Result.Categories; |
||||
|
C.bSexual = (*CategoriesObject)->GetBoolField(TEXT("sexual")); |
||||
|
C.bSexualMinors = (*CategoriesObject)->GetBoolField(TEXT("sexual/minors")); |
||||
|
C.bHarassment = (*CategoriesObject)->GetBoolField(TEXT("harassment")); |
||||
|
C.bHarassmentThreatening = (*CategoriesObject)->GetBoolField(TEXT("harassment/threatening")); |
||||
|
C.bHate = (*CategoriesObject)->GetBoolField(TEXT("hate")); |
||||
|
C.bHateThreatening = (*CategoriesObject)->GetBoolField(TEXT("hate/threatening")); |
||||
|
C.bIllicit = (*CategoriesObject)->GetBoolField(TEXT("illicit")); |
||||
|
C.bIllicitViolent = (*CategoriesObject)->GetBoolField(TEXT("illicit/violent")); |
||||
|
C.bSelfHarm = (*CategoriesObject)->GetBoolField(TEXT("self-harm")); |
||||
|
C.bSelfHarmIntent = (*CategoriesObject)->GetBoolField(TEXT("self-harm/intent")); |
||||
|
C.bSelfHarmInstructions = (*CategoriesObject)->GetBoolField(TEXT("self-harm/instructions")); |
||||
|
C.bViolence = (*CategoriesObject)->GetBoolField(TEXT("violence")); |
||||
|
C.bViolenceGraphic = (*CategoriesObject)->GetBoolField(TEXT("violence/graphic")); |
||||
|
} |
||||
|
|
||||
|
const TSharedPtr<FJsonObject>* ScoresObject; |
||||
|
if (FirstResult->TryGetObjectField(TEXT("category_scores"), ScoresObject)) |
||||
|
{ |
||||
|
FModerationCategoryScores& S = Result.CategoryScores; |
||||
|
S.Sexual = (*ScoresObject)->GetNumberField(TEXT("sexual")); |
||||
|
S.SexualMinors = (*ScoresObject)->GetNumberField(TEXT("sexual/minors")); |
||||
|
S.Harassment = (*ScoresObject)->GetNumberField(TEXT("harassment")); |
||||
|
S.HarassmentThreatening = (*ScoresObject)->GetNumberField(TEXT("harassment/threatening")); |
||||
|
S.Hate = (*ScoresObject)->GetNumberField(TEXT("hate")); |
||||
|
S.HateThreatening = (*ScoresObject)->GetNumberField(TEXT("hate/threatening")); |
||||
|
S.Illicit = (*ScoresObject)->GetNumberField(TEXT("illicit")); |
||||
|
S.IllicitViolent = (*ScoresObject)->GetNumberField(TEXT("illicit/violent")); |
||||
|
S.SelfHarm = (*ScoresObject)->GetNumberField(TEXT("self-harm")); |
||||
|
S.SelfHarmIntent = (*ScoresObject)->GetNumberField(TEXT("self-harm/intent")); |
||||
|
S.SelfHarmInstructions = (*ScoresObject)->GetNumberField(TEXT("self-harm/instructions")); |
||||
|
S.Violence = (*ScoresObject)->GetNumberField(TEXT("violence")); |
||||
|
S.ViolenceGraphic = (*ScoresObject)->GetNumberField(TEXT("violence/graphic")); |
||||
|
} |
||||
|
|
||||
|
if (Result.bFlagged) |
||||
|
{ |
||||
|
Flagged.Broadcast(Result); |
||||
|
} |
||||
|
else |
||||
|
{ |
||||
|
Passed.Broadcast(Result); |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,298 @@ |
|||||
|
// Fill out your copyright notice in the Description page of Project Settings.
|
||||
|
|
||||
|
|
||||
|
#include "Tools/OpenAiResponder.h" |
||||
|
|
||||
|
#include "HttpModule.h" |
||||
|
#include "Interfaces/IHttpResponse.h" |
||||
|
#include "Dom/JsonObject.h" |
||||
|
#include "Serialization/JsonReader.h" |
||||
|
#include "Serialization/JsonSerializer.h" |
||||
|
#include "Serialization/JsonWriter.h" |
||||
|
|
||||
|
FString UOpenAiResponder::StoredApiKey; |
||||
|
FString UOpenAiResponder::StoredModel = TEXT("gpt-4.1"); |
||||
|
TArray<TWeakObjectPtr<UOpenAiResponder>> UOpenAiResponder::ActiveRequests; |
||||
|
|
||||
|
void UOpenAiResponder::InitResponder(const FString& ApiKey, const FString& Model) |
||||
|
{ |
||||
|
StoredApiKey = ApiKey; |
||||
|
StoredModel = Model; |
||||
|
} |
||||
|
|
||||
|
UOpenAiResponder* UOpenAiResponder::CallOpenAiResponse( |
||||
|
UObject* WorldContextObject, |
||||
|
const FString& Prompt, |
||||
|
const TArray<FResponderMCPTool>& MCPTools, |
||||
|
const FString& Instructions, |
||||
|
const FString& OverrideModel, |
||||
|
int32 MaxOutputTokens, |
||||
|
EResponderReasoning Reasoning, |
||||
|
bool bUseWebSearch, |
||||
|
EResponderToolChoice ToolChoice) |
||||
|
{ |
||||
|
UOpenAiResponder* Action = NewObject<UOpenAiResponder>(); |
||||
|
Action->InputPrompt = Prompt; |
||||
|
Action->InputInstructions = Instructions; |
||||
|
Action->InputModel = OverrideModel.IsEmpty() ? StoredModel : OverrideModel; |
||||
|
Action->InputMaxOutputTokens = MaxOutputTokens; |
||||
|
Action->InputReasoning = Reasoning; |
||||
|
Action->bInputUseWebSearch = bUseWebSearch; |
||||
|
Action->InputMCPTools = MCPTools; |
||||
|
Action->InputToolChoice = ToolChoice; |
||||
|
Action->RegisterWithGameInstance(WorldContextObject); |
||||
|
return Action; |
||||
|
} |
||||
|
|
||||
|
void UOpenAiResponder::Activate() |
||||
|
{ |
||||
|
if (StoredApiKey.IsEmpty()) |
||||
|
{ |
||||
|
UE_LOG(LogTemp, Error, TEXT("OpenAiResponder: API key not set. Call InitResponder first.")); |
||||
|
Failed.Broadcast(TEXT("API key not set. Call InitResponder first.")); |
||||
|
return; |
||||
|
} |
||||
|
|
||||
|
RequestStartTime = FPlatformTime::Seconds(); |
||||
|
ActiveRequests.Add(this); |
||||
|
|
||||
|
// ── Build JSON request body ────────────────────────────────────────────
|
||||
|
|
||||
|
TSharedRef<FJsonObject> RootObject = MakeShared<FJsonObject>(); |
||||
|
RootObject->SetStringField(TEXT("model"), InputModel); |
||||
|
RootObject->SetStringField(TEXT("input"), InputPrompt); |
||||
|
|
||||
|
if (!InputInstructions.IsEmpty()) |
||||
|
{ |
||||
|
RootObject->SetStringField(TEXT("instructions"), InputInstructions); |
||||
|
} |
||||
|
|
||||
|
if (InputMaxOutputTokens > 0) |
||||
|
{ |
||||
|
RootObject->SetNumberField(TEXT("max_output_tokens"), InputMaxOutputTokens); |
||||
|
} |
||||
|
|
||||
|
// Reasoning
|
||||
|
if (InputReasoning != EResponderReasoning::None) |
||||
|
{ |
||||
|
TSharedRef<FJsonObject> ReasoningObject = MakeShared<FJsonObject>(); |
||||
|
FString EffortStr; |
||||
|
switch (InputReasoning) |
||||
|
{ |
||||
|
case EResponderReasoning::Low: EffortStr = TEXT("low"); break; |
||||
|
case EResponderReasoning::Medium: EffortStr = TEXT("medium"); break; |
||||
|
case EResponderReasoning::High: EffortStr = TEXT("high"); break; |
||||
|
default: break; |
||||
|
} |
||||
|
ReasoningObject->SetStringField(TEXT("effort"), EffortStr); |
||||
|
RootObject->SetObjectField(TEXT("reasoning"), ReasoningObject); |
||||
|
} |
||||
|
|
||||
|
// Tools array
|
||||
|
const bool bHasTools = bInputUseWebSearch || InputMCPTools.Num() > 0; |
||||
|
if (bHasTools) |
||||
|
{ |
||||
|
TArray<TSharedPtr<FJsonValue>> ToolsArray; |
||||
|
|
||||
|
// Web search
|
||||
|
if (bInputUseWebSearch) |
||||
|
{ |
||||
|
TSharedRef<FJsonObject> WebSearchTool = MakeShared<FJsonObject>(); |
||||
|
WebSearchTool->SetStringField(TEXT("type"), TEXT("web_search_preview")); |
||||
|
ToolsArray.Add(MakeShared<FJsonValueObject>(WebSearchTool)); |
||||
|
} |
||||
|
|
||||
|
// MCP tools
|
||||
|
for (const FResponderMCPTool& MCPTool : InputMCPTools) |
||||
|
{ |
||||
|
TSharedRef<FJsonObject> MCPObject = MakeShared<FJsonObject>(); |
||||
|
MCPObject->SetStringField(TEXT("type"), TEXT("mcp")); |
||||
|
MCPObject->SetStringField(TEXT("server_label"), MCPTool.ServerLabel); |
||||
|
MCPObject->SetStringField(TEXT("server_url"), MCPTool.ServerUrl); |
||||
|
|
||||
|
if (MCPTool.Headers.Num() > 0) |
||||
|
{ |
||||
|
TSharedRef<FJsonObject> HeadersObject = MakeShared<FJsonObject>(); |
||||
|
for (const auto& Pair : MCPTool.Headers) |
||||
|
{ |
||||
|
HeadersObject->SetStringField(Pair.Key, Pair.Value); |
||||
|
} |
||||
|
MCPObject->SetObjectField(TEXT("headers"), HeadersObject); |
||||
|
} |
||||
|
|
||||
|
ToolsArray.Add(MakeShared<FJsonValueObject>(MCPObject)); |
||||
|
} |
||||
|
|
||||
|
RootObject->SetArrayField(TEXT("tools"), ToolsArray); |
||||
|
|
||||
|
// Tool choice: auto-promote None → Auto when tools are present
|
||||
|
EResponderToolChoice EffectiveToolChoice = InputToolChoice; |
||||
|
if (EffectiveToolChoice == EResponderToolChoice::None) |
||||
|
{ |
||||
|
EffectiveToolChoice = EResponderToolChoice::Auto; |
||||
|
} |
||||
|
|
||||
|
switch (EffectiveToolChoice) |
||||
|
{ |
||||
|
case EResponderToolChoice::Auto: |
||||
|
RootObject->SetStringField(TEXT("tool_choice"), TEXT("auto")); |
||||
|
break; |
||||
|
case EResponderToolChoice::Required: |
||||
|
RootObject->SetStringField(TEXT("tool_choice"), TEXT("required")); |
||||
|
break; |
||||
|
default: |
||||
|
break; |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
// Serialize to string
|
||||
|
FString Body; |
||||
|
TSharedRef<TJsonWriter<>> Writer = TJsonWriterFactory<>::Create(&Body); |
||||
|
FJsonSerializer::Serialize(RootObject, Writer); |
||||
|
|
||||
|
// ── Fire HTTP request ──────────────────────────────────────────────────
|
||||
|
|
||||
|
TSharedRef<IHttpRequest, ESPMode::ThreadSafe> HttpRequest = FHttpModule::Get().CreateRequest(); |
||||
|
HttpRequest->SetURL(TEXT("https://api.openai.com/v1/responses")); |
||||
|
HttpRequest->SetVerb(TEXT("POST")); |
||||
|
HttpRequest->SetHeader(TEXT("Content-Type"), TEXT("application/json")); |
||||
|
HttpRequest->SetHeader(TEXT("Authorization"), FString::Printf(TEXT("Bearer %s"), *StoredApiKey)); |
||||
|
HttpRequest->SetContentAsString(Body); |
||||
|
HttpRequest->OnProcessRequestComplete().BindUObject(this, &UOpenAiResponder::HandleHttpResponse); |
||||
|
|
||||
|
if (!HttpRequest->ProcessRequest()) |
||||
|
{ |
||||
|
UE_LOG(LogTemp, Error, TEXT("OpenAiResponder: Failed to start HTTP request.")); |
||||
|
Failed.Broadcast(TEXT("Failed to start HTTP request.")); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
void UOpenAiResponder::ClearAllResponses() |
||||
|
{ |
||||
|
TArray<TWeakObjectPtr<UOpenAiResponder>> RequestsCopy = ActiveRequests; |
||||
|
ActiveRequests.Empty(); |
||||
|
|
||||
|
for (const TWeakObjectPtr<UOpenAiResponder>& Weak : RequestsCopy) |
||||
|
{ |
||||
|
if (UOpenAiResponder* Action = Weak.Get()) |
||||
|
{ |
||||
|
Action->SetReadyToDestroy(); |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
void UOpenAiResponder::HandleHttpResponse(FHttpRequestPtr Request, FHttpResponsePtr Response, bool bWasSuccessful) |
||||
|
{ |
||||
|
ActiveRequests.RemoveAll([this](const TWeakObjectPtr<UOpenAiResponder>& Weak) |
||||
|
{ |
||||
|
return !Weak.IsValid() || Weak.Get() == this; |
||||
|
}); |
||||
|
|
||||
|
const float ElapsedTime = static_cast<float>(FPlatformTime::Seconds() - RequestStartTime); |
||||
|
|
||||
|
if (!bWasSuccessful || !Response.IsValid()) |
||||
|
{ |
||||
|
UE_LOG(LogTemp, Error, TEXT("OpenAiResponder: HTTP request failed.")); |
||||
|
Failed.Broadcast(TEXT("HTTP request failed.")); |
||||
|
return; |
||||
|
} |
||||
|
|
||||
|
const int32 Code = Response->GetResponseCode(); |
||||
|
const FString Body = Response->GetContentAsString(); |
||||
|
|
||||
|
if (Code < 200 || Code >= 300) |
||||
|
{ |
||||
|
UE_LOG(LogTemp, Error, TEXT("OpenAiResponder: HTTP %d — %s"), Code, *Body); |
||||
|
Failed.Broadcast(FString::Printf(TEXT("HTTP %d — %s"), Code, *Body)); |
||||
|
return; |
||||
|
} |
||||
|
|
||||
|
// Parse root JSON
|
||||
|
TSharedPtr<FJsonObject> RootObject; |
||||
|
TSharedRef<TJsonReader<>> Reader = TJsonReaderFactory<>::Create(Body); |
||||
|
if (!FJsonSerializer::Deserialize(Reader, RootObject) || !RootObject.IsValid()) |
||||
|
{ |
||||
|
UE_LOG(LogTemp, Error, TEXT("OpenAiResponder: Failed to parse response JSON.")); |
||||
|
Failed.Broadcast(TEXT("Failed to parse response JSON.")); |
||||
|
return; |
||||
|
} |
||||
|
|
||||
|
// Check for API-level error
|
||||
|
if (RootObject->HasField(TEXT("error"))) |
||||
|
{ |
||||
|
const TSharedPtr<FJsonObject>* ErrorObject; |
||||
|
if (RootObject->TryGetObjectField(TEXT("error"), ErrorObject)) |
||||
|
{ |
||||
|
FString ErrorMsg = (*ErrorObject)->GetStringField(TEXT("message")); |
||||
|
UE_LOG(LogTemp, Error, TEXT("OpenAiResponder: API error — %s"), *ErrorMsg); |
||||
|
Failed.Broadcast(ErrorMsg); |
||||
|
return; |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
// Navigate output[] → find message → content[]
|
||||
|
FResponderResult Result; |
||||
|
Result.ExecutionTime = ElapsedTime; |
||||
|
|
||||
|
const TArray<TSharedPtr<FJsonValue>>* OutputArray; |
||||
|
if (!RootObject->TryGetArrayField(TEXT("output"), OutputArray)) |
||||
|
{ |
||||
|
UE_LOG(LogTemp, Error, TEXT("OpenAiResponder: No output array in response.")); |
||||
|
Failed.Broadcast(TEXT("No output array in response.")); |
||||
|
return; |
||||
|
} |
||||
|
|
||||
|
for (const TSharedPtr<FJsonValue>& OutputItem : *OutputArray) |
||||
|
{ |
||||
|
const TSharedPtr<FJsonObject>& OutputObject = OutputItem->AsObject(); |
||||
|
if (!OutputObject.IsValid()) continue; |
||||
|
|
||||
|
FString ItemType = OutputObject->GetStringField(TEXT("type")); |
||||
|
if (ItemType != TEXT("message")) continue; |
||||
|
|
||||
|
const TArray<TSharedPtr<FJsonValue>>* ContentArray; |
||||
|
if (!OutputObject->TryGetArrayField(TEXT("content"), ContentArray)) continue; |
||||
|
|
||||
|
for (const TSharedPtr<FJsonValue>& ContentItem : *ContentArray) |
||||
|
{ |
||||
|
const TSharedPtr<FJsonObject>& ContentObject = ContentItem->AsObject(); |
||||
|
if (!ContentObject.IsValid()) continue; |
||||
|
|
||||
|
FResponseContent ResponseContent; |
||||
|
ResponseContent.Type = ContentObject->GetStringField(TEXT("type")); |
||||
|
ResponseContent.Text = ContentObject->HasField(TEXT("text")) ? ContentObject->GetStringField(TEXT("text")) : FString(); |
||||
|
|
||||
|
// Parse annotations
|
||||
|
const TArray<TSharedPtr<FJsonValue>>* AnnotationsArray; |
||||
|
if (ContentObject->TryGetArrayField(TEXT("annotations"), AnnotationsArray)) |
||||
|
{ |
||||
|
for (const TSharedPtr<FJsonValue>& AnnotationItem : *AnnotationsArray) |
||||
|
{ |
||||
|
const TSharedPtr<FJsonObject>& AnnotationObject = AnnotationItem->AsObject(); |
||||
|
if (!AnnotationObject.IsValid()) continue; |
||||
|
|
||||
|
FResponseAnnotation Annotation; |
||||
|
Annotation.Type = AnnotationObject->GetStringField(TEXT("type")); |
||||
|
Annotation.StartIndex = static_cast<int32>(AnnotationObject->GetNumberField(TEXT("start_index"))); |
||||
|
Annotation.EndIndex = static_cast<int32>(AnnotationObject->GetNumberField(TEXT("end_index"))); |
||||
|
Annotation.Url = AnnotationObject->HasField(TEXT("url")) ? AnnotationObject->GetStringField(TEXT("url")) : FString(); |
||||
|
Annotation.Title = AnnotationObject->HasField(TEXT("title")) ? AnnotationObject->GetStringField(TEXT("title")) : FString(); |
||||
|
|
||||
|
ResponseContent.Annotations.Add(Annotation); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
Result.Content.Add(ResponseContent); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
if (Result.Content.Num() > 0) |
||||
|
{ |
||||
|
Success.Broadcast(Result); |
||||
|
} |
||||
|
else |
||||
|
{ |
||||
|
UE_LOG(LogTemp, Warning, TEXT("OpenAiResponder: Response contained no message content.")); |
||||
|
Failed.Broadcast(TEXT("Response contained no message content.")); |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,159 @@ |
|||||
|
// Fill out your copyright notice in the Description page of Project Settings.
|
||||
|
|
||||
|
#pragma once |
||||
|
|
||||
|
#include "CoreMinimal.h" |
||||
|
#include "Kismet/BlueprintAsyncActionBase.h" |
||||
|
#include "Http.h" |
||||
|
#include "OpenAiModerator.generated.h" |
||||
|
|
||||
|
USTRUCT(BlueprintType) |
||||
|
struct FModerationCategories |
||||
|
{ |
||||
|
GENERATED_BODY() |
||||
|
|
||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation") |
||||
|
bool bSexual = false; |
||||
|
|
||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation") |
||||
|
bool bSexualMinors = false; |
||||
|
|
||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation") |
||||
|
bool bHarassment = false; |
||||
|
|
||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation") |
||||
|
bool bHarassmentThreatening = false; |
||||
|
|
||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation") |
||||
|
bool bHate = false; |
||||
|
|
||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation") |
||||
|
bool bHateThreatening = false; |
||||
|
|
||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation") |
||||
|
bool bIllicit = false; |
||||
|
|
||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation") |
||||
|
bool bIllicitViolent = false; |
||||
|
|
||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation") |
||||
|
bool bSelfHarm = false; |
||||
|
|
||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation") |
||||
|
bool bSelfHarmIntent = false; |
||||
|
|
||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation") |
||||
|
bool bSelfHarmInstructions = false; |
||||
|
|
||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation") |
||||
|
bool bViolence = false; |
||||
|
|
||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation") |
||||
|
bool bViolenceGraphic = false; |
||||
|
}; |
||||
|
|
||||
|
USTRUCT(BlueprintType) |
||||
|
struct FModerationCategoryScores |
||||
|
{ |
||||
|
GENERATED_BODY() |
||||
|
|
||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation") |
||||
|
float Sexual = 0.f; |
||||
|
|
||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation") |
||||
|
float SexualMinors = 0.f; |
||||
|
|
||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation") |
||||
|
float Harassment = 0.f; |
||||
|
|
||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation") |
||||
|
float HarassmentThreatening = 0.f; |
||||
|
|
||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation") |
||||
|
float Hate = 0.f; |
||||
|
|
||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation") |
||||
|
float HateThreatening = 0.f; |
||||
|
|
||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation") |
||||
|
float Illicit = 0.f; |
||||
|
|
||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation") |
||||
|
float IllicitViolent = 0.f; |
||||
|
|
||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation") |
||||
|
float SelfHarm = 0.f; |
||||
|
|
||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation") |
||||
|
float SelfHarmIntent = 0.f; |
||||
|
|
||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation") |
||||
|
float SelfHarmInstructions = 0.f; |
||||
|
|
||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation") |
||||
|
float Violence = 0.f; |
||||
|
|
||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation") |
||||
|
float ViolenceGraphic = 0.f; |
||||
|
}; |
||||
|
|
||||
|
USTRUCT(BlueprintType) |
||||
|
struct FModerationResult |
||||
|
{ |
||||
|
GENERATED_BODY() |
||||
|
|
||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation") |
||||
|
bool bFlagged = false; |
||||
|
|
||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation") |
||||
|
FModerationCategories Categories; |
||||
|
|
||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation") |
||||
|
FModerationCategoryScores CategoryScores; |
||||
|
|
||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Moderation") |
||||
|
float ExecutionTime = 0.f; |
||||
|
}; |
||||
|
|
||||
|
DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FOnModerationResult, const FModerationResult&, Result); |
||||
|
|
||||
|
/**
|
||||
|
* Async Blueprint node for OpenAI Moderation API. |
||||
|
* Call InitModeration once to set API key and model, then use UseModeration to classify text. |
||||
|
*/ |
||||
|
UCLASS() |
||||
|
class AVATARCORE_AI_API UOpenAiModerator : public UBlueprintAsyncActionBase |
||||
|
{ |
||||
|
GENERATED_BODY() |
||||
|
|
||||
|
public: |
||||
|
/** Set API key and model. Call once before using UseModeration. */ |
||||
|
UFUNCTION(BlueprintCallable, Category = "AvatarCoreAI|Moderation", meta = (DisplayName = "Init Moderation")) |
||||
|
static void InitModeration(const FString& ApiKey, const FString& Model = TEXT("omni-moderation-latest")); |
||||
|
|
||||
|
/** Classify text using the OpenAI Moderation API. Returns via Flagged or Passed exec pins. */ |
||||
|
UFUNCTION(BlueprintCallable, Category = "AvatarCoreAI|Moderation", meta = (BlueprintInternalUseOnly = "true", DisplayName = "Use Moderation", WorldContext = "WorldContextObject")) |
||||
|
static UOpenAiModerator* UseModeration(UObject* WorldContextObject, const FString& Text); |
||||
|
|
||||
|
/** Cancel all active moderation requests. */ |
||||
|
UFUNCTION(BlueprintCallable, Category = "AvatarCoreAI|Moderation", meta = (DisplayName = "Clear Moderation")) |
||||
|
static void ClearModeration(); |
||||
|
|
||||
|
UPROPERTY(BlueprintAssignable) |
||||
|
FOnModerationResult Flagged; |
||||
|
|
||||
|
UPROPERTY(BlueprintAssignable) |
||||
|
FOnModerationResult Passed; |
||||
|
|
||||
|
virtual void Activate() override; |
||||
|
|
||||
|
private: |
||||
|
void HandleHttpResponse(FHttpRequestPtr Request, FHttpResponsePtr Response, bool bWasSuccessful); |
||||
|
|
||||
|
FString InputText; |
||||
|
double RequestStartTime = 0.0; |
||||
|
|
||||
|
static FString StoredApiKey; |
||||
|
static FString StoredModel; |
||||
|
static TArray<TWeakObjectPtr<UOpenAiModerator>> ActiveRequests; |
||||
|
}; |
||||
@ -0,0 +1,154 @@ |
|||||
|
// Fill out your copyright notice in the Description page of Project Settings.
|
||||
|
|
||||
|
#pragma once |
||||
|
|
||||
|
#include "CoreMinimal.h" |
||||
|
#include "Kismet/BlueprintAsyncActionBase.h" |
||||
|
#include "Http.h" |
||||
|
#include "OpenAiResponder.generated.h" |
||||
|
|
||||
|
// ── Enums ──────────────────────────────────────────────────────────────────
|
||||
|
|
||||
|
UENUM(BlueprintType) |
||||
|
enum class EResponderToolChoice : uint8 |
||||
|
{ |
||||
|
None UMETA(DisplayName = "None"), |
||||
|
Auto UMETA(DisplayName = "Auto"), |
||||
|
Required UMETA(DisplayName = "Required") |
||||
|
}; |
||||
|
|
||||
|
UENUM(BlueprintType) |
||||
|
enum class EResponderReasoning : uint8 |
||||
|
{ |
||||
|
None UMETA(DisplayName = "None"), |
||||
|
Low UMETA(DisplayName = "Low"), |
||||
|
Medium UMETA(DisplayName = "Medium"), |
||||
|
High UMETA(DisplayName = "High") |
||||
|
}; |
||||
|
|
||||
|
// ── Structs ────────────────────────────────────────────────────────────────
|
||||
|
|
||||
|
USTRUCT(BlueprintType) |
||||
|
struct FResponderMCPTool |
||||
|
{ |
||||
|
GENERATED_BODY() |
||||
|
|
||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Responder") |
||||
|
FString ServerLabel; |
||||
|
|
||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Responder") |
||||
|
FString ServerUrl; |
||||
|
|
||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Responder") |
||||
|
TMap<FString, FString> Headers; |
||||
|
}; |
||||
|
|
||||
|
USTRUCT(BlueprintType) |
||||
|
struct FResponseAnnotation |
||||
|
{ |
||||
|
GENERATED_BODY() |
||||
|
|
||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Responder") |
||||
|
FString Type; |
||||
|
|
||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Responder") |
||||
|
int32 StartIndex = 0; |
||||
|
|
||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Responder") |
||||
|
int32 EndIndex = 0; |
||||
|
|
||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Responder") |
||||
|
FString Url; |
||||
|
|
||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Responder") |
||||
|
FString Title; |
||||
|
}; |
||||
|
|
||||
|
USTRUCT(BlueprintType) |
||||
|
struct FResponseContent |
||||
|
{ |
||||
|
GENERATED_BODY() |
||||
|
|
||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Responder") |
||||
|
FString Type; |
||||
|
|
||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Responder") |
||||
|
FString Text; |
||||
|
|
||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Responder") |
||||
|
TArray<FResponseAnnotation> Annotations; |
||||
|
}; |
||||
|
|
||||
|
USTRUCT(BlueprintType) |
||||
|
struct FResponderResult |
||||
|
{ |
||||
|
GENERATED_BODY() |
||||
|
|
||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Responder") |
||||
|
TArray<FResponseContent> Content; |
||||
|
|
||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "AvatarCoreAI|Responder") |
||||
|
float ExecutionTime = 0.f; |
||||
|
}; |
||||
|
|
||||
|
// ── Delegates ──────────────────────────────────────────────────────────────
|
||||
|
|
||||
|
DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FOnResponderSuccess, const FResponderResult&, Result); |
||||
|
DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FOnResponderFailed, const FString&, ErrorMessage); |
||||
|
|
||||
|
// ── Class ──────────────────────────────────────────────────────────────────
|
||||
|
|
||||
|
UCLASS() |
||||
|
class AVATARCORE_AI_API UOpenAiResponder : public UBlueprintAsyncActionBase |
||||
|
{ |
||||
|
GENERATED_BODY() |
||||
|
|
||||
|
public: |
||||
|
/** Set API key and default model. Call once before using CallOpenAiResponse. */ |
||||
|
UFUNCTION(BlueprintCallable, Category = "AvatarCoreAI|Responder", meta = (DisplayName = "Init Responder")) |
||||
|
static void InitResponder(const FString& ApiKey, const FString& Model = TEXT("gpt-4.1")); |
||||
|
|
||||
|
/** Send a one-shot request to the OpenAI Responses API. */ |
||||
|
UFUNCTION(BlueprintCallable, Category = "AvatarCoreAI|Responder", |
||||
|
meta = (BlueprintInternalUseOnly = "true", DisplayName = "Call OpenAI Response", WorldContext = "WorldContextObject", |
||||
|
AutoCreateRefTerm = "Instructions,OverrideModel,MCPTools")) |
||||
|
static UOpenAiResponder* CallOpenAiResponse( |
||||
|
UObject* WorldContextObject, |
||||
|
const FString& Prompt, |
||||
|
const TArray<FResponderMCPTool>& MCPTools, |
||||
|
const FString& Instructions, |
||||
|
const FString& OverrideModel, |
||||
|
int32 MaxOutputTokens = 0, |
||||
|
EResponderReasoning Reasoning = EResponderReasoning::None, |
||||
|
bool bUseWebSearch = false, |
||||
|
EResponderToolChoice ToolChoice = EResponderToolChoice::None); |
||||
|
|
||||
|
/** Cancel all active response requests. */ |
||||
|
UFUNCTION(BlueprintCallable, Category = "AvatarCoreAI|Responder", meta = (DisplayName = "Clear All Responses")) |
||||
|
static void ClearAllResponses(); |
||||
|
|
||||
|
UPROPERTY(BlueprintAssignable) |
||||
|
FOnResponderSuccess Success; |
||||
|
|
||||
|
UPROPERTY(BlueprintAssignable) |
||||
|
FOnResponderFailed Failed; |
||||
|
|
||||
|
virtual void Activate() override; |
||||
|
|
||||
|
private: |
||||
|
void HandleHttpResponse(FHttpRequestPtr Request, FHttpResponsePtr Response, bool bWasSuccessful); |
||||
|
|
||||
|
FString InputPrompt; |
||||
|
FString InputInstructions; |
||||
|
FString InputModel; |
||||
|
int32 InputMaxOutputTokens = 0; |
||||
|
EResponderReasoning InputReasoning = EResponderReasoning::None; |
||||
|
bool bInputUseWebSearch = false; |
||||
|
TArray<FResponderMCPTool> InputMCPTools; |
||||
|
EResponderToolChoice InputToolChoice = EResponderToolChoice::None; |
||||
|
double RequestStartTime = 0.0; |
||||
|
|
||||
|
static FString StoredApiKey; |
||||
|
static FString StoredModel; |
||||
|
static TArray<TWeakObjectPtr<UOpenAiResponder>> ActiveRequests; |
||||
|
}; |
||||
@ -0,0 +1,24 @@ |
|||||
|
{ |
||||
|
"FileVersion": 3, |
||||
|
"Version": 1, |
||||
|
"VersionName": "1.0", |
||||
|
"FriendlyName": "AvatarCore_FaceDetector", |
||||
|
"Description": "Tracking faces in webcame streams", |
||||
|
"Category": "Other", |
||||
|
"CreatedBy": "b.ReX", |
||||
|
"CreatedByURL": "", |
||||
|
"DocsURL": "", |
||||
|
"MarketplaceURL": "", |
||||
|
"SupportURL": "", |
||||
|
"CanContainContent": true, |
||||
|
"IsBetaVersion": false, |
||||
|
"IsExperimentalVersion": false, |
||||
|
"Installed": false, |
||||
|
"Modules": [ |
||||
|
{ |
||||
|
"Name": "AvatarCore_FaceDetector", |
||||
|
"Type": "Runtime", |
||||
|
"LoadingPhase": "Default" |
||||
|
} |
||||
|
] |
||||
|
} |
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@ -0,0 +1,60 @@ |
|||||
|
// Copyright Epic Games, Inc. All Rights Reserved.
|
||||
|
|
||||
|
using System.IO; |
||||
|
using UnrealBuildTool; |
||||
|
|
||||
|
public class AvatarCore_FaceDetector : ModuleRules |
||||
|
{ |
||||
|
public AvatarCore_FaceDetector(ReadOnlyTargetRules Target) : base(Target) |
||||
|
{ |
||||
|
PCHUsage = ModuleRules.PCHUsageMode.UseExplicitOrSharedPCHs; |
||||
|
|
||||
|
PublicIncludePaths.AddRange( |
||||
|
new string[] { |
||||
|
// ... add public include paths required here ...
|
||||
|
} |
||||
|
); |
||||
|
|
||||
|
|
||||
|
PrivateIncludePaths.AddRange( |
||||
|
new string[] { |
||||
|
// ... add other private include paths required here ...
|
||||
|
} |
||||
|
); |
||||
|
|
||||
|
|
||||
|
PublicDependencyModuleNames.AddRange( |
||||
|
new string[] |
||||
|
{ |
||||
|
"Core", |
||||
|
// ... add other public dependencies that you statically link with here ...
|
||||
|
} |
||||
|
); |
||||
|
|
||||
|
|
||||
|
PrivateDependencyModuleNames.AddRange( |
||||
|
new string[] |
||||
|
{ |
||||
|
"CoreUObject", |
||||
|
"Engine", |
||||
|
"Projects", |
||||
|
"Json", |
||||
|
"JsonUtilities", |
||||
|
"Slate", |
||||
|
"SlateCore", |
||||
|
// ... add private dependencies that you statically link with here ...
|
||||
|
} |
||||
|
); |
||||
|
|
||||
|
|
||||
|
DynamicallyLoadedModuleNames.AddRange( |
||||
|
new string[] |
||||
|
{ |
||||
|
// ... add any modules that your module loads dynamically here ...
|
||||
|
} |
||||
|
); |
||||
|
|
||||
|
RuntimeDependencies.Add(Path.Combine(ModuleDirectory, "..", "ThirdParty", "FaceDetector.exe")); |
||||
|
RuntimeDependencies.Add(Path.Combine(ModuleDirectory, "..", "ThirdParty", "face_landmarker.task")); |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,20 @@ |
|||||
|
// Copyright Epic Games, Inc. All Rights Reserved.
|
||||
|
|
||||
|
#include "AvatarCore_FaceDetector.h" |
||||
|
|
||||
|
#define LOCTEXT_NAMESPACE "FAvatarCore_FaceDetectorModule" |
||||
|
|
||||
|
void FAvatarCore_FaceDetectorModule::StartupModule() |
||||
|
{ |
||||
|
// This code will execute after your module is loaded into memory; the exact timing is specified in the .uplugin file per-module
|
||||
|
} |
||||
|
|
||||
|
void FAvatarCore_FaceDetectorModule::ShutdownModule() |
||||
|
{ |
||||
|
// This function may be called during shutdown to clean up your module. For modules that support dynamic reloading,
|
||||
|
// we call this function before unloading the module.
|
||||
|
} |
||||
|
|
||||
|
#undef LOCTEXT_NAMESPACE |
||||
|
|
||||
|
IMPLEMENT_MODULE(FAvatarCore_FaceDetectorModule, AvatarCore_FaceDetector) |
||||
@ -0,0 +1,307 @@ |
|||||
|
#include "AvatarFaceDetectorSubsystem.h" |
||||
|
|
||||
|
#include "Async/Async.h" |
||||
|
#include "Interfaces/IPluginManager.h" |
||||
|
#include "Misc/Paths.h" |
||||
|
#include "Serialization/JsonReader.h" |
||||
|
#include "Serialization/JsonSerializer.h" |
||||
|
|
||||
|
bool UAvatarFaceDetectorSubsystem::InitializeFaceDetector(const FFaceDetectorSettings& Settings) |
||||
|
{ |
||||
|
ShutdownFaceDetector(); |
||||
|
return StartProcess(Settings); |
||||
|
} |
||||
|
|
||||
|
void UAvatarFaceDetectorSubsystem::ShutdownFaceDetector() |
||||
|
{ |
||||
|
StopProcess(); |
||||
|
} |
||||
|
|
||||
|
bool UAvatarFaceDetectorSubsystem::IsFaceDetectorRunning() const |
||||
|
{ |
||||
|
FProcHandle HandleCopy = ProcHandle; |
||||
|
return HandleCopy.IsValid() && FPlatformProcess::IsProcRunning(HandleCopy); |
||||
|
} |
||||
|
|
||||
|
void UAvatarFaceDetectorSubsystem::Deinitialize() |
||||
|
{ |
||||
|
ShutdownFaceDetector(); |
||||
|
Super::Deinitialize(); |
||||
|
} |
||||
|
|
||||
|
FString UAvatarFaceDetectorSubsystem::ResolveThirdPartyFilePath(const FString& FileName) const |
||||
|
{ |
||||
|
const TSharedPtr<IPlugin> Plugin = IPluginManager::Get().FindPlugin(TEXT("AvatarCore_FaceDetector")); |
||||
|
if (!Plugin.IsValid()) |
||||
|
{ |
||||
|
return FString(); |
||||
|
} |
||||
|
|
||||
|
const FString ThirdPartyDir = FPaths::Combine(Plugin->GetBaseDir(), TEXT("Source"), TEXT("ThirdParty")); |
||||
|
return FPaths::Combine(ThirdPartyDir, FileName); |
||||
|
} |
||||
|
|
||||
|
bool UAvatarFaceDetectorSubsystem::StartProcess(const FFaceDetectorSettings& Settings) |
||||
|
{ |
||||
|
FScopeLock Lock(&StateCriticalSection); |
||||
|
|
||||
|
const FString ExePath = ResolveThirdPartyFilePath(TEXT("FaceDetector.exe")); |
||||
|
if (ExePath.IsEmpty() || !FPaths::FileExists(ExePath)) |
||||
|
{ |
||||
|
AsyncTask(ENamedThreads::GameThread, [this]() |
||||
|
{ |
||||
|
OnFaceDetectorError.Broadcast(TEXT("FaceDetector.exe not found (expected under Plugin/Source/ThirdParty).")); |
||||
|
}); |
||||
|
return false; |
||||
|
} |
||||
|
|
||||
|
FString Args; |
||||
|
if (!Settings.WebcamName.IsEmpty()) |
||||
|
{ |
||||
|
Args += FString::Printf(TEXT(" --device \"%s\""), *Settings.WebcamName); |
||||
|
} |
||||
|
if (!Settings.ModelName.IsEmpty()) |
||||
|
{ |
||||
|
const FString ModelPath = ResolveThirdPartyFilePath(Settings.ModelName); |
||||
|
Args += FString::Printf(TEXT(" --model \"%s\""), *ModelPath); |
||||
|
} |
||||
|
Args += FString::Printf(TEXT(" --num_faces %d"), Settings.NumFaces); |
||||
|
if (Settings.bDebugMode) |
||||
|
{ |
||||
|
Args += TEXT(" --debugMode"); |
||||
|
} |
||||
|
|
||||
|
Args += FString::Printf(TEXT(" --fps_limit %d"), Settings.FrameRateLimit); |
||||
|
|
||||
|
FPlatformProcess::CreatePipe(ReadPipe, WritePipe); |
||||
|
|
||||
|
bStopRequested = false; |
||||
|
StdoutRemainder.Reset(); |
||||
|
|
||||
|
ProcHandle = FPlatformProcess::CreateProc( |
||||
|
*ExePath, |
||||
|
*Args, |
||||
|
false, |
||||
|
true, |
||||
|
true, |
||||
|
nullptr, |
||||
|
0, |
||||
|
nullptr, |
||||
|
WritePipe, |
||||
|
ReadPipe); |
||||
|
|
||||
|
if (!ProcHandle.IsValid()) |
||||
|
{ |
||||
|
FPlatformProcess::ClosePipe(ReadPipe, WritePipe); |
||||
|
ReadPipe = nullptr; |
||||
|
WritePipe = nullptr; |
||||
|
|
||||
|
AsyncTask(ENamedThreads::GameThread, [this]() |
||||
|
{ |
||||
|
OnFaceDetectorError.Broadcast(TEXT("Failed to launch FaceDetector.exe")); |
||||
|
}); |
||||
|
return false; |
||||
|
} |
||||
|
|
||||
|
ReaderFuture = Async(EAsyncExecution::Thread, [this]() |
||||
|
{ |
||||
|
ReaderThreadMain(); |
||||
|
}); |
||||
|
|
||||
|
return true; |
||||
|
} |
||||
|
|
||||
|
void UAvatarFaceDetectorSubsystem::StopProcess() |
||||
|
{ |
||||
|
TFuture<void> LocalFuture; |
||||
|
FProcHandle LocalProc; |
||||
|
void* LocalReadPipe = nullptr; |
||||
|
void* LocalWritePipe = nullptr; |
||||
|
|
||||
|
{ |
||||
|
FScopeLock Lock(&StateCriticalSection); |
||||
|
|
||||
|
bStopRequested = true; |
||||
|
LocalFuture = MoveTemp(ReaderFuture); |
||||
|
LocalProc = ProcHandle; |
||||
|
LocalReadPipe = ReadPipe; |
||||
|
LocalWritePipe = WritePipe; |
||||
|
|
||||
|
ProcHandle.Reset(); |
||||
|
ReadPipe = nullptr; |
||||
|
WritePipe = nullptr; |
||||
|
StdoutRemainder.Reset(); |
||||
|
} |
||||
|
|
||||
|
if (LocalProc.IsValid()) |
||||
|
{ |
||||
|
FPlatformProcess::TerminateProc(LocalProc, true); |
||||
|
FPlatformProcess::CloseProc(LocalProc); |
||||
|
} |
||||
|
|
||||
|
if (LocalFuture.IsValid()) |
||||
|
{ |
||||
|
LocalFuture.Wait(); |
||||
|
} |
||||
|
|
||||
|
if (LocalReadPipe != nullptr || LocalWritePipe != nullptr) |
||||
|
{ |
||||
|
FPlatformProcess::ClosePipe(LocalReadPipe, LocalWritePipe); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
void UAvatarFaceDetectorSubsystem::ReaderThreadMain() |
||||
|
{ |
||||
|
while (!bStopRequested) |
||||
|
{ |
||||
|
FString NewOutput; |
||||
|
{ |
||||
|
FScopeLock Lock(&StateCriticalSection); |
||||
|
if (ReadPipe == nullptr) |
||||
|
{ |
||||
|
break; |
||||
|
} |
||||
|
NewOutput = FPlatformProcess::ReadPipe(ReadPipe); |
||||
|
} |
||||
|
|
||||
|
if (NewOutput.IsEmpty()) |
||||
|
{ |
||||
|
if (!IsFaceDetectorRunning()) |
||||
|
{ |
||||
|
break; |
||||
|
} |
||||
|
FPlatformProcess::Sleep(0.01f); |
||||
|
continue; |
||||
|
} |
||||
|
|
||||
|
FString Combined; |
||||
|
{ |
||||
|
FScopeLock Lock(&StateCriticalSection); |
||||
|
Combined = StdoutRemainder + NewOutput; |
||||
|
StdoutRemainder.Reset(); |
||||
|
} |
||||
|
|
||||
|
TArray<FString> Lines; |
||||
|
Combined.ParseIntoArrayLines(Lines, false); |
||||
|
|
||||
|
const bool bEndsWithNewline = Combined.EndsWith(TEXT("\n")) || Combined.EndsWith(TEXT("\r\n")); |
||||
|
if (!bEndsWithNewline && Lines.Num() > 0) |
||||
|
{ |
||||
|
FScopeLock Lock(&StateCriticalSection); |
||||
|
StdoutRemainder = Lines.Pop(); |
||||
|
} |
||||
|
|
||||
|
for (const FString& Line : Lines) |
||||
|
{ |
||||
|
const FString Trimmed = Line.TrimStartAndEnd(); |
||||
|
if (!Trimmed.IsEmpty()) |
||||
|
{ |
||||
|
HandleLineOnWorkerThread(Trimmed); |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
if (!bStopRequested) |
||||
|
{ |
||||
|
AsyncTask(ENamedThreads::GameThread, [this]() |
||||
|
{ |
||||
|
OnFaceDetectorError.Broadcast(TEXT("FaceDetector process stopped.")); |
||||
|
}); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
void UAvatarFaceDetectorSubsystem::HandleLineOnWorkerThread(const FString& Line) |
||||
|
{ |
||||
|
TSharedPtr<FJsonObject> Root; |
||||
|
TSharedRef<TJsonReader<>> Reader = TJsonReaderFactory<>::Create(Line); |
||||
|
if (!FJsonSerializer::Deserialize(Reader, Root) || !Root.IsValid()) |
||||
|
{ |
||||
|
AsyncTask(ENamedThreads::GameThread, [this, Line]() |
||||
|
{ |
||||
|
OnFaceDetectorLog.Broadcast(Line); |
||||
|
}); |
||||
|
return; |
||||
|
} |
||||
|
|
||||
|
FString Type; |
||||
|
if (!Root->TryGetStringField(TEXT("type"), Type)) |
||||
|
{ |
||||
|
AsyncTask(ENamedThreads::GameThread, [this, Line]() |
||||
|
{ |
||||
|
OnFaceDetectorLog.Broadcast(Line); |
||||
|
}); |
||||
|
return; |
||||
|
} |
||||
|
|
||||
|
if (Type.Equals(TEXT("error"), ESearchCase::IgnoreCase)) |
||||
|
{ |
||||
|
FString Message; |
||||
|
Root->TryGetStringField(TEXT("message"), Message); |
||||
|
AsyncTask(ENamedThreads::GameThread, [this, Message]() |
||||
|
{ |
||||
|
OnFaceDetectorError.Broadcast(Message); |
||||
|
}); |
||||
|
return; |
||||
|
} |
||||
|
|
||||
|
if (Type.Equals(TEXT("log"), ESearchCase::IgnoreCase)) |
||||
|
{ |
||||
|
FString Message; |
||||
|
Root->TryGetStringField(TEXT("message"), Message); |
||||
|
AsyncTask(ENamedThreads::GameThread, [this, Message]() |
||||
|
{ |
||||
|
OnFaceDetectorLog.Broadcast(Message); |
||||
|
}); |
||||
|
return; |
||||
|
} |
||||
|
|
||||
|
if (Type.Equals(TEXT("faces"), ESearchCase::IgnoreCase)) |
||||
|
{ |
||||
|
TArray<FDetectedFace> Faces; |
||||
|
|
||||
|
const TArray<TSharedPtr<FJsonValue>>* FacesJson = nullptr; |
||||
|
if (Root->TryGetArrayField(TEXT("faces"), FacesJson) && FacesJson != nullptr) |
||||
|
{ |
||||
|
Faces.Reserve(FacesJson->Num()); |
||||
|
for (const TSharedPtr<FJsonValue>& FaceVal : *FacesJson) |
||||
|
{ |
||||
|
if (!FaceVal.IsValid() || FaceVal->Type != EJson::Object) |
||||
|
{ |
||||
|
continue; |
||||
|
} |
||||
|
const TSharedPtr<FJsonObject> FaceObj = FaceVal->AsObject(); |
||||
|
if (!FaceObj.IsValid()) |
||||
|
{ |
||||
|
continue; |
||||
|
} |
||||
|
|
||||
|
double X = 0.0; |
||||
|
double Y = 0.0; |
||||
|
double IdNum = (double)INDEX_NONE; |
||||
|
bool bActive = false; |
||||
|
|
||||
|
FaceObj->TryGetNumberField(TEXT("x"), X); |
||||
|
FaceObj->TryGetNumberField(TEXT("y"), Y); |
||||
|
FaceObj->TryGetNumberField(TEXT("id"), IdNum); |
||||
|
FaceObj->TryGetBoolField(TEXT("active"), bActive); |
||||
|
|
||||
|
FDetectedFace Face; |
||||
|
Face.Position = FVector2D((float)X, (float)Y); |
||||
|
Face.Id = (int32)IdNum; |
||||
|
Face.TrackingLost = !bActive; |
||||
|
Faces.Add(Face); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
AsyncTask(ENamedThreads::GameThread, [this, Faces = MoveTemp(Faces)]() mutable |
||||
|
{ |
||||
|
OnFaceDetectorUpdate.Broadcast(Faces); |
||||
|
}); |
||||
|
return; |
||||
|
} |
||||
|
|
||||
|
AsyncTask(ENamedThreads::GameThread, [this, Line]() |
||||
|
{ |
||||
|
OnFaceDetectorLog.Broadcast(Line); |
||||
|
}); |
||||
|
} |
||||
@ -0,0 +1,14 @@ |
|||||
|
// Copyright Epic Games, Inc. All Rights Reserved.
|
||||
|
|
||||
|
#pragma once |
||||
|
|
||||
|
#include "Modules/ModuleManager.h" |
||||
|
|
||||
|
class FAvatarCore_FaceDetectorModule : public IModuleInterface |
||||
|
{ |
||||
|
public: |
||||
|
|
||||
|
/** IModuleInterface implementation */ |
||||
|
virtual void StartupModule() override; |
||||
|
virtual void ShutdownModule() override; |
||||
|
}; |
||||
@ -0,0 +1,56 @@ |
|||||
|
#pragma once |
||||
|
|
||||
|
#include "CoreMinimal.h" |
||||
|
#include "HAL/PlatformProcess.h" |
||||
|
#include "Async/Future.h" |
||||
|
#include "Subsystems/GameInstanceSubsystem.h" |
||||
|
#include "FaceDetectorTypes.h" |
||||
|
#include "AvatarFaceDetectorSubsystem.generated.h" |
||||
|
|
||||
|
DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FFaceDetectorErrorSig, const FString&, ErrorMessage); |
||||
|
DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FFaceDetectorLogSig, const FString&, LogMessage); |
||||
|
DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FFaceDetectorUpdateSig, const TArray<FDetectedFace>&, Faces); |
||||
|
|
||||
|
UCLASS() |
||||
|
class AVATARCORE_FACEDETECTOR_API UAvatarFaceDetectorSubsystem : public UGameInstanceSubsystem |
||||
|
{ |
||||
|
GENERATED_BODY() |
||||
|
|
||||
|
public: |
||||
|
UFUNCTION(BlueprintCallable, Category="FaceDetector") |
||||
|
bool InitializeFaceDetector(const FFaceDetectorSettings& Settings); |
||||
|
|
||||
|
UFUNCTION(BlueprintCallable, Category="FaceDetector") |
||||
|
void ShutdownFaceDetector(); |
||||
|
|
||||
|
UFUNCTION(BlueprintCallable, Category="FaceDetector") |
||||
|
bool IsFaceDetectorRunning() const; |
||||
|
|
||||
|
UPROPERTY(BlueprintAssignable, Category="FaceDetector") |
||||
|
FFaceDetectorErrorSig OnFaceDetectorError; |
||||
|
|
||||
|
UPROPERTY(BlueprintAssignable, Category="FaceDetector") |
||||
|
FFaceDetectorLogSig OnFaceDetectorLog; |
||||
|
|
||||
|
UPROPERTY(BlueprintAssignable, Category="FaceDetector") |
||||
|
FFaceDetectorUpdateSig OnFaceDetectorUpdate; |
||||
|
|
||||
|
virtual void Deinitialize() override; |
||||
|
|
||||
|
private: |
||||
|
bool StartProcess(const FFaceDetectorSettings& Settings); |
||||
|
void StopProcess(); |
||||
|
void ReaderThreadMain(); |
||||
|
void HandleLineOnWorkerThread(const FString& Line); |
||||
|
|
||||
|
FString ResolveThirdPartyFilePath(const FString& FileName) const; |
||||
|
|
||||
|
private: |
||||
|
FProcHandle ProcHandle; |
||||
|
void* ReadPipe = nullptr; |
||||
|
void* WritePipe = nullptr; |
||||
|
TAtomic<bool> bStopRequested = false; |
||||
|
TFuture<void> ReaderFuture; |
||||
|
FCriticalSection StateCriticalSection; |
||||
|
FString StdoutRemainder; |
||||
|
}; |
||||
@ -0,0 +1,45 @@ |
|||||
|
#pragma once |
||||
|
|
||||
|
#include "CoreMinimal.h" |
||||
|
#include "FaceDetectorTypes.generated.h" |
||||
|
|
||||
|
USTRUCT(BlueprintType) |
||||
|
struct FDetectedFace |
||||
|
{ |
||||
|
GENERATED_BODY() |
||||
|
|
||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category="FaceDetector") |
||||
|
FVector2D Position = FVector2D::ZeroVector; |
||||
|
|
||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category="FaceDetector") |
||||
|
int32 Id = INDEX_NONE; |
||||
|
|
||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category="FaceDetector") |
||||
|
bool TrackingLost = false; |
||||
|
}; |
||||
|
|
||||
|
USTRUCT(BlueprintType) |
||||
|
struct FFaceDetectorSettings |
||||
|
{ |
||||
|
GENERATED_BODY() |
||||
|
|
||||
|
//Name of the webcam (pick first if ambigious)
|
||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category="FaceDetector") |
||||
|
FString WebcamName; |
||||
|
|
||||
|
//Create a DebugWindow to see detected faces
|
||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category="FaceDetector") |
||||
|
bool bDebugMode = false; |
||||
|
|
||||
|
//Override Model (full path to modelfile)
|
||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category="FaceDetector") |
||||
|
FString ModelName; |
||||
|
|
||||
|
//Limit Framerate of Videostream (0 to disable)
|
||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category="FaceDetector") |
||||
|
int32 FrameRateLimit = 24; |
||||
|
|
||||
|
//Max faces to track
|
||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category="FaceDetector") |
||||
|
int32 NumFaces = 5; |
||||
|
}; |
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Some files were not shown because too many files changed in this diff
Loading…
Reference in new issue