Unfortunately, NNE currently only supports floats as in and output types, thus text is not yet supported.
But we are working on it, please stay tuned for updates on this!
Ears peeled to the ground for updates on text support!
Thanks so much for this effort, Nico. Pleased to see Epic supporting this tech so soon, and the community providing an entry point on how to use it! Hungry for more!
Hi there,
Thanks very much for the plugin! It looks promising.
I tried to load some “onnx” files and create a model with the “RDG” runtime.
Firstly, the model could not be created because of the invalid “Transpose” operator in the model.
Then, after we trimmed the “Transpose” operator, the model still cannot be optimized because of the “Resize” operator.
You can see the log here:
LogTemp: Warning: RDG runtime available: NNERuntimeRDGHlsl
LogTemp: Warning: RDG runtime available: NNERuntimeRDGDml
LogTemp: Warning: GPU runtime available: NNERuntimeORTCuda
LogTemp: Warning: GPU runtime available: NNERuntimeORTDml
LogTemp: Warning: CPU runtime available: NNERuntimeORTCpu
LogTemp: Display: Succeeded in creating a model on GPU
LogNNE: Warning: RDG MLOperator:Resize is not registered
LogNNE: Warning: RDG MLOperatorRegistry failed to find validation for operator:Resize
LogNNE: Warning: Model validator RDG Model validator detected an error.
LogNNE: Warning: Model is not valid, skipping optimization passes.
I am just wondering to meet the validity to run with “RDG”, what kind of model should be imported?
Thanks very much and have a good day!
Hi @MioSPE ,
Unfortunately not all runtimes support all operators yet, sorry! This is work in progress and we continuously try to extend them. Are you trying to run on RDGDml or RDGHlsl? If RDGHlsl, maybe try out Dml as it has more operators available.
Sorry for the inconvenience, please stay tuned for future releases with wider model support.
Hi, I followed the tutorial. I am trying to implement the CPU runtime with a model I built in Keras.
I have converted it to an ONNX file, and confirmed the ONNX model was predicting the correct class, but when I run inference in UE5 it always returns category 25 (out of 29 categories).
I am a C++ / NNE beginner. Here is my code. Please let me know if this you see any errors.
// 5: hexagon
TArray<TArray<TArray<float>>> Hexagon = {{
{0.05490196, 0.0 },
{0.17254902, 0.02745098},
{0.34901961, 0.04313725},
{0.62352941, 0.04313725},
{1.0, 0.64705882},
{0.76862745, 0.77254902},
{0.60784314, 0.91764706},
{0.13333333, 0.90980392},
{0.0, 0.54509804},
{0.0,0.0},
{0.0,0.0},
{0.0,0.0},
{0.0,0.0},
{0.0,0.0},
{0.0,0.0},
{0.0,0.0},
{0.0,0.0},
{0.0,0.0},
{0.0,0.0},
{0.0,0.0},
{0.0,0.0},
{0.0,0.0},
{0.0,0.0},
{0.0,0.0},
{0.0,0.0},
{0.0,0.0},
{0.0,0.0},
{0.0,0.0},
{0.0,0.0},
{0.0,0.0},
{0.0,0.0},
{0.0,0.0},
{0.0,0.0},
{0.0,0.0},
{0.0,0.0},
{0.0,0.0},
{0.0,0.0},
{0.0,0.0},
{0.0,0.0},
{0.0,0.0}}};
TArray<TArray<TArray<float>>> Squiggle = {{
{0.0,0.16078431}, // squiggle / item 26
{0.03137255, 0.10588235},
{0.08235294, 0.05490196},
{0.13333333, 0.05882353},
{0.23137255, 0.1254902 },
{0.27058824, 0.13333333},
{0.30588235, 0.11372549},
{0.36078431, 0.03921569},
{0.42745098, 0.01176471},
{0.50196078, 0.03921569},
{0.58431373, 0.12156863},
{0.62745098, 0.14117647},
{0.6745098, 0.11372549},
{0.72941176, 0.02352941},
{0.79215686, 0.0},
{0.91764706, 0.03921569},
{0.96470588, 0.08627451},
{1.0, 0.14901961},
{0.0,0.0},
{0.0,0.0},
{0.0,0.0},
{0.0,0.0},
{0.0,0.0},
{0.0,0.0},
{0.0,0.0},
{0.0,0.0},
{0.0,0.0},
{0.0,0.0},
{0.0,0.0},
{0.0,0.0},
{0.0,0.0},
{0.0,0.0},
{0.0,0.0},
{0.0,0.0},
{0.0,0.0},
{0.0,0.0},
{0.0,0.0},
{0.0,0.0},
{0.0,0.0},
{0.0,0.0}}};
// 18: 'tornado'
TArray<TArray<TArray<float>>> Tornado = {{
{ 0.0, 0.0 },
{ 0.04313725, 0.02352941},
{ 0.20784314, 0.0627451 },
{ 0.54117647, 0.08235294},
{ 0.86666667, 0.0745098 },
{ 0.23137255, 0.11372549},
{ 0.08627451, 0.14901961},
{ 0.90980392, 0.15686275},
{ 0.78431373, 0.18431373},
{ 0.14509804, 0.21960784},
{ 0.19607843, 0.23921569},
{ 0.4, 0.27058824},
{ 0.91372549, 0.29411765},
{ 0.68627451, 0.31764706},
{ 0.19607843, 0.33333333},
{ 0.28235294, 0.35686275},
{ 0.76078431, 0.40784314},
{ 0.75294118, 0.41960784},
{ 0.70196078, 0.43137255},
{ 0.42745098, 0.44705882},
{ 0.5372549, 0.49803922},
{ 0.78431373, 0.54509804},
{ 0.50196078, 0.6 },
{ 0.53333333, 0.62745098},
{ 0.82745098, 0.7254902 },
{ 0.6, 0.76078431},
{ 0.61960784, 0.78039216},
{ 0.81568627, 0.80784314},
{ 0.60392157, 0.85490196},
{ 0.7372549, 0.92156863},
{ 0.67843137, 0.94901961},
{ 0.71372549, 0.95294118},
{ 0.66666667, 0.98039216},
{ 0.68627451, 1.0 },
{ 0.0, 0.0 },
{ 0.0, 0.0 },
{ 0.0, 0.0 },
{ 0.0, 0.0 },
{ 0.0, 0.0 },
{ 0.0, 0.0 }
}};
TWeakInterfacePtr<INNERuntimeCPU> Runtime = UE::NNE::GetRuntime<INNERuntimeCPU>(FString("NNERuntimeORTCpu"));
if (Runtime.IsValid())
{
TUniquePtr<UE::NNE::IModelCPU> Model = Runtime->CreateModel(PreLoadedModelData);
if (Model.IsValid())
{
TUniquePtr<UE::NNE::IModelInstanceCPU> ModelInstance = Model->CreateModelInstance();
if (ModelInstance.IsValid())
{
TConstArrayView<uint32> InputDims = { 1, 40, 2 };
TConstArrayView<uint32> OutputDims = { 29 };
TArray<UE::NNE::FTensorShape> InputTensorShapes = { UE::NNE::FTensorShape::Make(InputDims) };
ModelInstance->SetInputTensorShapes(InputTensorShapes);
TArray<UE::NNE::FTensorShape> OutputTensorShapes = { UE::NNE::FTensorShape::Make(OutputDims) };
TArray<TArray<TArray<float>>> InputData = Tornado;
TArray<float> OutputData;
TArray<UE::NNE::FTensorBindingCPU> InputBindings;
TArray<UE::NNE::FTensorBindingCPU> OutputBindings;
InputData.SetNumZeroed(InputTensorShapes[0].Volume());
InputBindings.SetNumZeroed(1);
InputBindings[0].Data = InputData.GetData();
InputBindings[0].SizeInBytes = InputData.Num() * sizeof(float);
OutputData.SetNumZeroed(OutputTensorShapes[0].Volume());
OutputBindings.SetNumZeroed(1);
OutputBindings[0].Data = OutputData.GetData();
OutputBindings[0].SizeInBytes = OutputData.Num() * sizeof(float);
if (ModelInstance->RunSync(InputBindings, OutputBindings) != 0) {
UE_LOG(LogTemp, Error, TEXT("Failed to run the model"));
}
float max = 0;
int j = 0;
for (int i=0; i < OutputData.Num(); i++){
UE_LOG(LogTemp, Warning, TEXT("%d"), i);
UE_LOG(LogTemp, Warning, TEXT("%f"), OutputData[i]);
if (max < OutputData[i]){
max = OutputData[i];
j = i;
}
}
UE_LOG(LogTemp, Warning, TEXT("Output: %f"), max);
UE_LOG(LogTemp, Warning, TEXT("Output: %d"), j);
}
else
{
UE_LOG(LogTemp, Error, TEXT("Failed to create the model instance"));
}
}
else
{
UE_LOG(LogTemp, Error, TEXT("Failed to create the model"));
}
}
else
{
UE_LOG(LogTemp, Error, TEXT("Cannot find runtime NNERuntimeORTCpu, please enable the corresponding plugin"));
}
Hey @jmccrory13,
Few things that could cause the issue:
First, you need to put the data into one continuous memory block.
So do not use nested arrays, but one single array with Volume() elements inside.
Secondly, you can make InputDims TArray, it will automatically convert it to a const view when passed to FTensorShapeMake.
Then be carful with setting up the bindings:
Currently you assign Tornado to InputData and later InputData.SetNumZeroed which will erase the content you will later pass to the network.
Also, by assigning Tornado to InputData you make an expensive copy and you could just use a const view instead.
So I would try the following:
TArray Squiggle = { … flattened data … };
TArray Tornado = { … flattened data … };TConstArrayView InputData = Tornado.GetData(); // This makes a view without a copy
TArrayUE::NNE::FTensorShape InputTensorShapes = { UE::NNE::FTensorShape::Make({ 1, 40, 2 }) };
ModelInstance->SetInputTensorShapes(InputTensorShapes);
TArrayUE::NNE::FTensorShape OutputTensorShapes = { UE::NNE::FTensorShape::Make({ 29 }) };TArrayUE::NNE::FTensorBindingCPU InputBindings;
InputBindings.SetNumZeroed(1);
InputBindings[0].Data = InputData.GetData();
InputBindings[0].SizeInBytes = InputData.Num() * sizeof(float);TArray OutputData;
OutputData.SetNumUninitialized(OutputTensorShapes[0].Volume());TArrayUE::NNE::FTensorBindingCPU OutputBindings;
OutputBindings.SetNumZeroed(1);
OutputBindings[0].Data = OutputData.GetData();
OutputBindings[0].SizeInBytes = OutputData.Num() * sizeof(float);
Let me know if this works 
Thank you!!! 
I couldn’t get TConstArrayView to work, but flattening the array worked 
!
Great, glad to hear!
I just see a mistake in my pseudo code:
TConstArrayView InputData = Tornado.GetData();
should be
TConstArrayView<float> InputData = Tornado;
or something alike, sorry for that
Hi. I’d like to use NNE for Style Transfers.
I think inferencing with RDG model is going work because GPU Time is increasing when executing RunInferenceRDG().
The current problem is how to get the output data in the form of TArray<float>.
According to previous posts on this forum, it seems that an external buffer needs to be extracted and copied into memory. I’ve been trying to do this for a couple of days now, but I can’t. I’m so tired right now.
I’ve read the RDG documentation of course, but I’m still at a loss.
I would be grateful if you could tell me how to convert the output to TArray<float> in this code.
Thank you.
RDGModel.cpp (8.1 KB)
This tutorial was mentioned above, but there is a tutorial for style transfer in UE5
The mentioned tutorial was copying a frame back to cpu, do inference there and uploading again if I remember correctly.The power of the RDG runtimes is, that it is executed as part of rendering a frame, without any CPU-sync.
In this particular case you could inherit and implement FSceneViewExtensionBase, which will call you (PrePostProcessPass_RenderThread) when you can EnqueueRDG a neural network with the final rendered frame as input (inside FPostProcessingInputs). Of course you would need to write some shaders that first convert the texture to a FRDGBuffer containing floats, and convert the result back to a texture. But its possible (and certainly fun ^^).
However, I have good news for you here (You need access to the Epic github for this link). It is a plugin where you can easily add style transfer (or any other neural post processing) through a special material node.
Thank you, I had a fundamental misunderstanding about the RDG runtime.
And thanks for the awsome plugin introduction.
It’s given me some inspiration to move forward with my work. I’ll try to work on it a bit more.
Thanks very much! I appreciate your quick response.
I tried to load the model with “NNERuntimeRDGDml” runtime, it crashed the UE project.
However, the simple “mnist-8.onnx” model can be loaded with no issue.
Also, if I want to use the current rendered frame in the editor as the input, is there an API call that I can use to get the buffer?
Yes NNERuntimeRDGDml has some issues we are working on. You can try to use NNERuntimeRDGHlsl instead, it is more stable but has less operators implemented (yet) and thus you may not be able to run complex models.
Regarding frame input: Check out my last answer, that would be the approach with the FSceneViewExtensionBase. In the mentioned callback, you get all input buffers of the final rendered frame in the clabback argument. SO the idea is, that not you call the render thread, but the render thread calls you when the buffers are ready.
Dear all,
What an amasing package. I am currently following the tutorial on having the NNE on a single actor, link at: NNE - Quick Start Guide - 5.2 | Tutorial.
I am able to run this 80% of this tutorial calling the cpp functions in blueprints, apart from setting inputs and outputs when running the game. This happens when calling the function SetInputs()
Everything loads, such as “pointilism-8” onnx model (can’t get mnist or candy8 but can also instantate vgg16 actually) but when creating inputs and outputs I get: LogNNE: Error: Run(): Input shapes are not set, please call SetInputTensorShapes. Follows some prints, the “Print String” after SetInputs() is the only false in the chain.
This is the only thing I am stuck, everything seems to load on runtime which is amasing. Just bump into this (UE 5.2 too, compilation on visual studio 2022 no errors).
Thanks in advance
Hey @joaquincortez
A neural network needs some internal memory to be evaluated (think of buffers where intermediate results are stored). The buffer sizes depend on the input shapes and as a neural network could have dynamic shapes, NNE requires the model’s SetInputShape method to be called at least once before the first inference and then every time you change the input size. In you case the solution is simple: When you setup the network and have the shapes ready, just call SetInputShapes on your model. If you call your SetInputs node only once, it could go in there. Hope that helped!
Thanks very much @ranierin
It is working. Had to do some logic which seems to me a bit forcing the event begin but leave here for reference. I just had a bug but I believe it is related to the runtime when I stick an evaluation to the event tick. Here follows the function I am using to log the input space:
bool UNeuralNetworkModel::SetNetInputs(const TArray<FNeuralNetworkTensor>& Inputs)
{
check(Model.IsValid())
using namespace UE::NNECore;
InputBindings.Reset();
InputShapes.Reset();
TConstArrayView<FTensorDesc> InputDescs = Model->GetInputTensorDescs();
if (InputDescs.Num() != Inputs.Num())
{
UE_LOG(LogTemp, Error, TEXT("Invalid number of input tensors provided"));
return false;
}
// Additional logging to diagnose input shapes and data
UE_LOG(LogTemp, Warning, TEXT("Number of Input Tensors: %d"), Inputs.Num());
for (int32 i = 0; i < Inputs.Num(); ++i)
{
const FNeuralNetworkTensor& Tensor = Inputs[i];
// Get the expected shape of the tensor
TArray<int32> ExpectedShape = GetInputShape(i);
// Log the expected shape for debugging
FString ExpectedShapeString;
for (int32 Dimension : ExpectedShape)
{
ExpectedShapeString += FString::Printf(TEXT("%d "), Dimension);
}
UE_LOG(LogTemp, Warning, TEXT("Expected Shape for Input Tensor %d: %s"), i, *ExpectedShapeString);
}
// Same as in tutorial, ...
Now the bug I get from visual studio is a link issue perhaps.
Thanks in advance for the help in establishing this. I believe this can be a huge thing for ue5, for real.
Good work so far!
I think you are missing a return statement in UNeuralNetworkModel but that should not be the cause of the issue.
Then it says ‘Access violation writingt 0x0’ which could be an indication that something went wrong when setting up your outputs.
Remember: You must make sure that your memory (e.g. TArray) remain valid throughout model evaluation, you as a caller own the memory.
The same holds for the input/output bindings and those contain just a pointer to your memory. So in principle you can reuse the same memory and do not have to set the inputs and outputs again, just overwrite the input array before running the model and read out the outputs after.
Thanks so much @ranierin.
I understood that part and it makes perfect sense to just initialise everything on startup and keep on filling that, now with this we have adaptible input and output spaces.
I put this to work just by doing exactly the same for outputs in the cpp code. I believe it is working and I can easily do 32fps when sticking a RunSync() on event tick with vgg-16. This is absolutely amasing, I hope you keep this library structure for the future!
Any ideas on where should I look to develop a method for prediction? Like a “model->predict()”. Should this not be designed and only update the “inputs” tensor variable in the game? I believe it might be possible to create a way to feed pixel distributions to model and get 1/1000 classes for classification.
Thanks.
Ref, something like:
bool UNeuralNetworkModel::SetNetOutputs(TArray<FNeuralNetworkTensor>& Outputs)
{
check(Model.IsValid())
using namespace UE::NNECore;
TConstArrayView<FTensorDesc> OutputDescs = Model->GetOutputTensorDescs();
UE_LOG(LogTemp, Warning, TEXT("Number of Output Tensors: %d"), Outputs.Num());
UE_LOG(LogTemp, Warning, TEXT("Number of Output Descs: %d"), OutputDescs.Num());
for (int32 i = 0; i < OutputDescs.Num(); ++i)
{
const UE::NNECore::FTensorDesc& TensorDesc = OutputDescs[i];
TArray<int32> ExpectedOutShape = GetOutputShape(i);
FString ShapeString;
for (int32 Dimension : ExpectedOutShape)
{
ShapeString += FString::Printf(TEXT("%d "), Dimension);
UE_LOG(LogTemp, Warning, TEXT("Expected Shape for Output Tensor %d: %s"), i, *ShapeString);
}
OutputBindings.SetNum(OutputDescs.Num());
OutputShapes.SetNum(OutputDescs.Num());
}
return true;
}
Not sure if I understand this, sorry. ‘predict’ typically refers to evaluating a neural network and computing the predictions given a set of input, which is exactly what RunSync does.
So if you setup your input buffers and output buffers, you can write your inputs, call RunSync and after you will find your predictions inside the output buffers.
Hey folks & @ranierin thanks for the intro here it has been immensely useful thus far. I have the inference system working well with pytorch (CNN) models that are exported to onnx format, however as soon as I use onnx tools to convert a lightgbm model to onnx format with the same classes / feature architecture I get a full exception to the point where I cannot discern any useful information about it ie:
LoginId:0654a4f3482454456d65a0918db7c7a5
EpicAccountId:<redacted>
Unhandled Exception: EXCEPTION_ACCESS_VIOLATION reading address 0x0000000000000000
UnrealEditor_NNEOnnxruntime
UnrealEditor_NNERuntimeORTCpu
UnrealEditor_NNERuntimeORTCpu
UnrealEditor_NNERuntimeORTCpu
UnrealEditor_DataMasterX_Win64_DebugGame!UNeuralData::InitModel() [D:\UnrealProjects\DataMasterX\Source\DataMasterX\Systems\NeuralData.cpp:17]
UnrealEditor_DataMasterX_Win64_DebugGame!UDataSubsystem::SetReplayMode() [D:\UnrealProjects\DataMasterX\Source\DataMasterX\Systems\DataSubSystem.cpp:39]
UnrealEditor_DataMasterX_Win64_DebugGame!USocketsSubsystem::Consume() [D:\UnrealProjects\DataMasterX\Source\DataMasterX\Systems\SocketsSubSystem.cpp:179]
UnrealEditor_DataMasterX_Win64_DebugGame!ADataController::BeginPlay() [D:\UnrealProjects\DataMasterX\Source\DataMasterX\Player\DataController.cpp:28]
UnrealEditor_Engine
UnrealEditor_Engine
UnrealEditor_Engine
UnrealEditor_Engine
UnrealEditor_Engine
UnrealEditor_Engine
UnrealEditor_Engine
UnrealEditor_Engine
UnrealEditor_UnrealEd
UnrealEditor_UnrealEd
UnrealEditor_UnrealEd
UnrealEditor_UnrealEd
UnrealEditor_UnrealEd
UnrealEditor_UnrealEd
UnrealEditor_UnrealEd
UnrealEditor_UnrealEd
UnrealEditor_Win64_DebugGame
UnrealEditor_Win64_DebugGame
That exception occurs during this function (Runtime->CreateModel(ModelData)->CreateModelInstance();):
bool UNeuralData::InitModel(const FString& ModelPath, const FString& RuntimeType)
{
ModelData = LoadObject<UNNEModelData>(GetTransientPackage(), *ModelPath);
Runtime = UE::NNE::GetRuntime<INNERuntimeCPU>(TEXT("NNERuntimeORTCpu"));
if (Runtime.IsValid())
{
try
{
ModelInstance = Runtime->CreateModel(ModelData)->CreateModelInstance();
if (!ModelInstance.IsValid())
{
UE_LOG(LogTemp, Error, TEXT("ModelInstance is invalid"));
return false;
}
}
catch (std::exception& e)
{
UE_LOG(LogTemp, Error, TEXT("Exception: %hs"), UTF8_TO_TCHAR(e.what()));
return false;
}
} else
{
UE_LOG(LogTemp, Error, TEXT("Invalid model runtime..."));
return false;
}
return true;
}
As you can see I’m still trying to wrap it in a try catch but that doesn’t work and I still see the exception. Happy to dive deeper as needed but wanted to see if anyone else has seen a similar issue. For review of the lightgbm setup it is effectively this:
model = lgb.LGBMClassifier(
num_leaves=args.num_leaves,
learning_rate=args.learning_rate,
n_estimators=args.n_estimators,
max_depth=args.max_depth,
subsample=args.subsample
)
... (MODEL Training) REDACTED FOR SPACE ...
model.booster_.save_model(ExportPath)
print(f"Model saved to {ExportPath} in LightGBM text format.")
output_onnx_model = f"{ExportPath}.onnx"
initial_types = [('feature_input', FloatTensorType([None, num_features]))]
onnx_model = onnxmltools.convert_lightgbm(model, initial_types=initial_types)
onnxmltools.utils.save_model(onnx_model, output_onnx_model)
print(f"Model saved to {ExportPath}.onnx in ONNX format.")
The exported lgbm works fine when put in inference mode as part of a inference http server. It also imports just fine into unreal engine and I can select the type just fine before saving the model in UE5.3. But beyond that I cannot get any deeper. I realize also that it was likely suppose to just support Neural networks but figured I’d try post here as there is no direct support for other models like XGBoost / LightGBM which I find far more useful especially for structured data which I generally create within unreal.
Thanks in advance.