I’ve been posting several separate posts all relating to the same project. I got pretty far but now am stuck on something. Memory Leaks!
Here’s a screen capture of where I’m at:
Functionality wise this is working! There are 4 image sequences on my desktop (so EXTERNAL to packaged game). This game is currently randomly setting in/out points on each of those clips, playing it, and then switching to the next clip and repeating that action over and over.
So for about a minute straight its working GREAT. Then at about 54 seconds, it freezes and then frame glitches like crazy! This coincides with my gpu usage hitting 30%ish each time it does that. I see the GPU % climb climb climb, hit 30% SCRAMBLE… then it drops way down to like 12%. then it slowly builds back up up up, hits 30% and SCRAMBLE.
So obviously how I’m doing this is bloating up the memory (computer gets HOT) and then it dies but recovers (maybe some automatic garbage collection repairs it?)
I figured this would be the case until I figured out how to have this stuff EXIT the memory.
Here’s the header file:
// Fill out your copyright notice in the Description page of Project Settings.
#pragma once
#include "CoreMinimal.h"
#include "GameFramework/Actor.h"
#include <Runtime\Core\Public\Misc\FileHelper.h>
#include <Runtime\ImageWrapper\Public\IImageWrapper.h>
#include <Runtime\ImageWrapper\Public\IImageWrapperModule.h>
#include "TEST_VideoStreamDisplay.generated.h"
UCLASS()
class IMAGETEXTURECPP_API ATEST_VideoStreamDisplay : public AActor
{
GENERATED_BODY()
public:
// Sets default values for this actor's properties
ATEST_VideoStreamDisplay();
FString ZeroFillInt32ToFString(int32 Value, int32 ZeroFillCount);
UFUNCTION()
void OnPlayNextFrame();
protected:
// Called when the game starts or when spawned
virtual void BeginPlay() override;
public:
// Called every frame
virtual void Tick(float DeltaTime) override;
UPROPERTY(EditAnywhere)
UStaticMeshComponent* SampleMesh;
UMaterialInstanceDynamic* MyMaterial;
UMaterialInterface* LoadedMaterial;
IImageWrapperModule& ImageWrapperModule = FModuleManager::LoadModuleChecked<IImageWrapperModule>(FName("ImageWrapper"));
TSharedPtr<IImageWrapper> PNGImageWrapper = ImageWrapperModule.CreateImageWrapper(EImageFormat::PNG);
//TArray<uint8> BinaryArray;
//TArray<uint8> RawData;
UTexture2D* myTexture;
int32 TEST_CurrentFrame = 0;
FString TEST_CurrentClipName = "earth";
int32 TEST_CurrentClipDuration = 707;
int32 TEST_CurrentOutPoint = 20;
};
and the implementation file
// Fill out your copyright notice in the Description page of Project Settings.
#include "TEST_VideoStreamDisplay.h"
#include <Runtime\Core\Public\Misc\Paths.h>
#include <Runtime\Core\Public\HAL\PlatformFilemanager.h>
#include <Runtime\ImageWrapper\Public\IImageWrapper.h>
#include <Runtime\ImageWrapper\Public\IImageWrapperModule.h>
// Sets default values
ATEST_VideoStreamDisplay::ATEST_VideoStreamDisplay()
{
// Set this actor to call Tick() every frame. You can turn this off to improve performance if you don't need it.
PrimaryActorTick.bCanEverTick = true;
SampleMesh = CreateDefaultSubobject<UStaticMeshComponent>(TEXT("SampleMesh"));
}
FString ATEST_VideoStreamDisplay::ZeroFillInt32ToFString(int32 Value, int32 ZeroFillCount)
{
FString InputValueStr = "";
InputValueStr.AppendInt(Value);
FString BuildReturnValue = "";
for (int32 i = InputValueStr.Len(); i < ZeroFillCount; i++)
{
BuildReturnValue += "0";
}
BuildReturnValue += InputValueStr;
return BuildReturnValue;
}
void ATEST_VideoStreamDisplay::OnPlayNextFrame()
{
FString ZeroFillString = ZeroFillInt32ToFString(TEST_CurrentFrame, 3);
//UE_LOG(LogTemp, Log, TEXT("ZeroFillString %s]"), *ZeroFillString);
FString PngFile = "C:/Users/dev/Desktop/BRYAN_TEST/"+TEST_CurrentClipName+"/"+TEST_CurrentClipName+"_";
PngFile += ZeroFillString;
PngFile += ".png";
//UE_LOG(LogTemp, Log, TEXT("PNG FILE %s]"), *PngFile);
//IImageWrapperModule& ImageWrapperModule = FModuleManager::LoadModuleChecked<IImageWrapperModule>(FName("ImageWrapper"));
//TSharedPtr<IImageWrapper> PNGImageWrapper = ImageWrapperModule.CreateImageWrapper(EImageFormat::PNG);
TArray<uint8> BinaryArray;
if (FFileHelper::LoadFileToArray(BinaryArray, *PngFile))
{
//UE_LOG(LogTemp, Log, TEXT("LoadFileToArray() success"));
if (PNGImageWrapper.IsValid() && PNGImageWrapper->SetCompressed(BinaryArray.GetData(), BinaryArray.Num()))
{
//UE_LOG(LogTemp, Log, TEXT("its valid, keep going"));
TArray<uint8> RawData;
if (PNGImageWrapper->GetRaw(ERGBFormat::BGRA, 8, RawData))
{
//UE_LOG(LogTemp, Log, TEXT("GetRaw success keep going"));
//UTexture2D* myTexture = UTexture2D::CreateTransient(600, 338, PF_B8G8R8A8);
myTexture = UTexture2D::CreateTransient(600, 338, PF_B8G8R8A8);
myTexture->UpdateResource();
//UE_LOG(LogTemp, Log, TEXT("Now try to make a material instance of the video stream material"));
//UMaterialInterface* LoadedMaterial = LoadObject<UMaterialInterface>(nullptr, TEXT("/Game/Foo.Foo"), nullptr, LOAD_None, nullptr);
// PUT_BACK if its not working!
if (LoadedMaterial)
{
//UE_LOG(LogTemp, Log, TEXT("LoadedMaterial true"));
//MyMaterial = UMaterialInstanceDynamic::Create(LoadedMaterial, NULL);
if (MyMaterial)
{
//UE_LOG(LogTemp, Log, TEXT("MyMaterial true"));
MyMaterial->SetTextureParameterValue(TEXT("TextureInput"), myTexture);
//Finally, apply the material to your mesh component
SampleMesh->SetMaterial(0, MyMaterial);
}
}
}
}
}
//up the current frame and run logic to switch to next clip
TEST_CurrentFrame++;
if (TEST_CurrentFrame > TEST_CurrentOutPoint) TEST_CurrentFrame = 9999;
if (TEST_CurrentFrame > TEST_CurrentClipDuration)
{
UE_LOG(LogTemp, Log, TEXT("time to change clips"));
TEST_CurrentFrame = 0;
if(TEST_CurrentClipName == "earth")
{
TEST_CurrentClipName = "fire";
TEST_CurrentClipDuration = 647;
}
else if (TEST_CurrentClipName == "fire")
{
TEST_CurrentClipName = "water";
TEST_CurrentClipDuration = 738;
}
else if (TEST_CurrentClipName == "water")
{
TEST_CurrentClipName = "wind";
TEST_CurrentClipDuration = 198;
}
else if (TEST_CurrentClipName == "wind")
{
TEST_CurrentClipName = "earth";
TEST_CurrentClipDuration = 707;
}
TEST_CurrentFrame = FMath::RandRange(int32(0), TEST_CurrentClipDuration);
//make the shot last up to 6 seconds
TEST_CurrentOutPoint = FMath::RandRange(TEST_CurrentFrame + 1, TEST_CurrentFrame+144);
//UE_LOG(LogTemp, Log, TEXT("Random Number: %d"), inPointFrame);
}
}
// Called when the game starts or when spawned
void ATEST_VideoStreamDisplay::BeginPlay()
{
Super::BeginPlay();
//Setup the SampleMesh position and scale for this video playback to display on
FRotator InitPlaneRotation = FRotator(0, 0, 0);
InitPlaneRotation.Pitch = 0;
InitPlaneRotation.Yaw = 90;
InitPlaneRotation.Roll = 90;
SampleMesh->SetRelativeRotation(InitPlaneRotation);
SampleMesh->SetRelativeScale3D(FVector(3.2, 1.8, 1));
//setup material stuff
LoadedMaterial = LoadObject<UMaterialInterface>(nullptr, TEXT("/Game/Foo.Foo"), nullptr, LOAD_None, nullptr);
MyMaterial = UMaterialInstanceDynamic::Create(LoadedMaterial, NULL);
//Setup the 24 FPS timer for video playback to run on, 0.0416666666666667 = 24 FPS time (approx)
//need to set up a delta time and make corrections for the incorrect timing of each function call.
FTimerHandle TimerHandle;
FTimerDelegate TimerDel;
TimerDel.BindUFunction(this, FName("OnPlayNextFrame"));
GetWorld()->GetTimerManager().SetTimer(TimerHandle, TimerDel, 0.0416666666666667, true);
}
// Called every frame
void ATEST_VideoStreamDisplay::Tick(float DeltaTime)
{
Super::Tick(DeltaTime);
}
Note: I know there are media textures that can load image sequences but i don’t think that route will work as I need very accurate start / stop, and to be abel to with custom code control the playback speed/rate at a greater degree than I found with the media texture route. Plus these are external not packaged.
Any tips/feedback would be greatly appreciated! Also the route of loading in a new texture onto that material instance 24 times a second I figure I’m doing wrong and that it’s just bloating memory or somehow creating a BUNCH of texture instances that bloat gpu memory somewhere.
If someone knows a better route to rapidly update the texture based on this attempt at radpily loading new video frames into the material I’m all ears!