Scenario:
I’m trying to combine the Kinect4Unreal plugin with OpenCV to do blob detection for interactive particle effects but running into some issues with getting the Kinect image into a texture I can process with OpenCV.
Kinect4Unreal doesn’t have source code so its part is in Blueprint. The plugin outputs a Texture2D reference that I pass to a C++ function as a UTexture2D*. Then I followed part of the Procedural Materials tutorial ( A new, community-hosted Unreal Engine Wiki - Announcements - Epic Developer Community Forums ) to get the pixels of my texture which then I pass onto OpenCV and then write the result to a destination texture applied to a 2nd mesh.
The issue:
When passing the Texture2D reference from the Kinect to my C++ function all I get in the destination texture is a solid gray colour. If I pass an actual texture from the Content Library, that works and I can manipulate it with OpenCV and display the result without issues.
Some code:
#include "OpenCVTest.h"
#include "KinectViewer.h"
#include "StaticMeshResources.h"
// Use this function to update the texture rects you want to change:
// NOTE: There is a method called UpdateTextureRegions in UTexture2D but it is compiled WITH_EDITOR and is not marked as ENGINE_API so it cannot be linked
// from plugins.
void UpdateTextureRegions(UTexture2D* Texture, int32 MipIndex, uint32 NumRegions, FUpdateTextureRegion2D* Regions, uint32 SrcPitch, uint32 SrcBpp, uint8* SrcData, bool bFreeData)
{
if (Texture->Resource)
{
struct FUpdateTextureRegionsData
{
FTexture2DResource* Texture2DResource;
int32 MipIndex;
uint32 NumRegions;
FUpdateTextureRegion2D* Regions;
uint32 SrcPitch;
uint32 SrcBpp;
uint8* SrcData;
};
FUpdateTextureRegionsData* RegionData = new FUpdateTextureRegionsData;
RegionData->Texture2DResource = (FTexture2DResource*)Texture->Resource;
RegionData->MipIndex = MipIndex;
RegionData->NumRegions = NumRegions;
RegionData->Regions = Regions;
RegionData->SrcPitch = SrcPitch;
RegionData->SrcBpp = SrcBpp;
RegionData->SrcData = SrcData;
ENQUEUE_UNIQUE_RENDER_COMMAND_TWOPARAMETER(
UpdateTextureRegionsData,
FUpdateTextureRegionsData*, RegionData, RegionData,
bool, bFreeData, bFreeData,
{
for (uint32 RegionIndex = 0; RegionIndex < RegionData->NumRegions; ++RegionIndex)
{
int32 CurrentFirstMip = RegionData->Texture2DResource->GetCurrentFirstMip();
if (RegionData->MipIndex >= CurrentFirstMip)
{
RHIUpdateTexture2D(
RegionData->Texture2DResource->GetTexture2DRHI(),
RegionData->MipIndex - CurrentFirstMip,
RegionData->Regions[RegionIndex],
RegionData->SrcPitch,
RegionData->SrcData
+ RegionData->Regions[RegionIndex].SrcY * RegionData->SrcPitch
+ RegionData->Regions[RegionIndex].SrcX * RegionData->SrcBpp
);
}
}
if (bFreeData)
{
FMemory::Free(RegionData->Regions);
FMemory::Free(RegionData->SrcData);
}
delete RegionData;
});
}
}
// Sets default values
AKinectViewer::AKinectViewer()
{
// Set this actor to call Tick() every frame. You can turn this off to improve performance if you don't need it.
PrimaryActorTick.bCanEverTick = true;
SourceMesh = CreateDefaultSubobject<UStaticMeshComponent>(TEXT("SourceMesh"));
RootComponent = SourceMesh;
DestinationMesh = CreateDefaultSubobject<UStaticMeshComponent>(TEXT("DestinationMesh"));
DestinationMesh->AttachTo(RootComponent);
}
// Called when the game starts or when spawned
void AKinectViewer::BeginPlay()
{
Super::BeginPlay();
}
void AKinectViewer::PostInitializeComponents()
{
Super::PostInitializeComponents();
}
// Called every frame
void AKinectViewer::Tick( float DeltaTime )
{
Super::Tick( DeltaTime );
UpdateTextureRegions(mDynamicTexture, 0, 1, mUpdateTextureRegion, mDataSqrtSize, (uint32)4, mDynamicColors, false);
// If I change this to mSourceTexture I can see the Kinect image replicated on the 2nd mesh
mDynamicMaterials[0]->SetTextureParameterValue("DestinationTexture", mDynamicTexture);
}
// Called from Blueprint after the Kinect is ready and a KinectImage is available
void AKinectViewer::SetSourceTextureReference(UTexture2D* KinectImage)
{
SizeX = 512;
SizeY = 424;
//Convert the static material in our mesh into a dynamic one, and store it (please note that if you have more than one material that you wish to mark dynamic, do so here).
mDynamicMaterials.Add(DestinationMesh->CreateAndSetMaterialInstanceDynamic(0));
//Create a dynamic texture with the default compression (B8G8R8A8)
mDynamicTexture = UTexture2D::CreateTransient(SizeX, SizeY);
//Make sure it won't be compressed
mDynamicTexture->CompressionSettings = TextureCompressionSettings::TC_VectorDisplacementmap;
//Turn off Gamma-correction
mDynamicTexture->SRGB = 0;
//Guarantee no garbage collection by adding it as a root reference
mDynamicTexture->AddToRoot();
//Update the texture with new variable values.
mDynamicTexture->UpdateResource();
mSourceTexture = KinectImage;
//Make sure it won't be compressed
mSourceTexture->CompressionSettings = TextureCompressionSettings::TC_VectorDisplacementmap;
//Turn off Gamma-correction
mSourceTexture->SRGB = 0;
//Guarantee no garbage collection by adding it as a root reference
mSourceTexture->AddToRoot();
//Update the texture with new variable values.
mSourceTexture->UpdateResource();
//Grab the colorvalues from our existing texture (the one we created at '''Texture Setup''') and copy it into a uint8* mTextureColors variable.
int32 w, h;
w = mSourceTexture->GetSizeX();
h = mSourceTexture->GetSizeY();
FTexture2DMipMap& readMip = mSourceTexture->PlatformData->Mips[0];
mDataSize = w * h * 4; // * 4 because we're working with uint8's - which are 4 bytes large
mDataSqrtSize = w * 4; // * 4 because we're working with uint8's - which are 4 bytes large
readMip.BulkData.GetCopy((void**)&mTextureColors);
// Initalize our dynamic pixel array with data size
mDynamicColors = new uint8[mDataSize];
// Copy our current texture's colors into our dynamic colors
FMemory::Memcpy(mDynamicColors, mTextureColors, mDataSize);
// Create a new texture region with the width and height of our dynamic texture
mUpdateTextureRegion = new FUpdateTextureRegion2D(0, 0, 0, 0, w, h);
// Set the Paramater in our material to our texture
mDynamicMaterials[0]->SetTextureParameterValue("DestinationTexture", mDynamicTexture);
}
void AKinectViewer::ProcessKinectImage()
{
// Some image processing tests
//int pixelAmount = mDataSize / 4;
//for (int i = 0; i < pixelAmount; ++i)
//{
// //int blue = i * 4 + 0;
// //int green = i * 4 + 1;
// //int red = i * 4 + 2;
// //int alpha = i * 4 + 3;
// mDynamicColors[i] = 120; // Set pixel's red value to 120
//}
//auto thresh = cv::Mat(SizeX, SizeY, CV_8UC4, mSourceTextureColors);
//cv::threshold(thresh, thresh, 0, 255, 3);
//cv::erode(thresh, thresh, cv::Mat());
//cv::dilate(thresh, thresh, cv::Mat());
//mDynamicColors = (uint8*)thresh.data;
}
The result (Kinect source image on the left, C++ result on the right):
Any ideas?
Thanks you in advance