I’m working on creating a segmentation dataset in Unreal Engine using Scene Capture Components and Render Targets. While I’ve successfully implemented the basic setup, I’m encountering a misalignment problem between the captured images and their corresponding semantic labeled versions. This misalignment seems to occur due to the time difference between when the original image is captured and when the segmentation is applied. Has anyone else faced a similar issue or found a solution to ensure perfect alignment between the captured scene and its semantic segmentation? I’m particularly interested in methods to minimize or eliminate this time gap to achieve more accurate results. Any advice or suggestions would be greatly appreciated!
Also I Tried:
UKismetRenderingLibrary::DrawMaterialToRenderTarget(GetWorld(), ActualRenderTarget, PPMI_Mask);
However the labeled -segmented- images are blank red.
void UStreamCaptureManager::SaveSegmentationTargets(USceneCaptureComponent2D* ActualCapture, USceneCaptureComponent2D* SegmentationCapture, EStreamDirection StreamDirection, const FString& ID)
{
UTextureRenderTarget2D* ActualRenderTarget = ActualCapture->TextureTarget;
FTextureRenderTargetResource* ActualTargetResource = ActualRenderTarget->GameThread_GetRenderTargetResource();
UTextureRenderTarget2D* SegmentationRenderTarget = SegmentationCapture->TextureTarget;
FTextureRenderTargetResource* SegmentationTargetResource = SegmentationRenderTarget->GameThread_GetRenderTargetResource();
FString ImagesFolderName = FString("images");
FString SegFolderName = FString("segmentation");
FString Extension = FString("png");
FString ImageFileName = GetSaveName(StreamDirection, ID, Extension, false);
FString SegFileName = GetSaveName(StreamDirection, ID, Extension, true);
TArray<FColor> ActualRawData;
ActualTargetResource->ReadPixels(ActualRawData);
TArray<FColor> SegmentationRawData;
SegmentationTargetResource->ReadPixels(SegmentationRawData);
const FString& ActualSaveFilePath = FPaths::Combine(FPaths::ProjectSavedDir(), TEXT("Screenshots"), *ImagesFolderName, *ImageFileName);
const FString& SegmentationSaveFilePath = FPaths::Combine(FPaths::ProjectSavedDir(), TEXT("Screenshots"), *SegFolderName, *SegFileName);
IPlatformFile& PlatformFile = FPlatformFileManager::Get().GetPlatformFile();
PlatformFile.CreateDirectoryTree(*FPaths::GetPath(ActualSaveFilePath));
PlatformFile.CreateDirectoryTree(*FPaths::GetPath(SegmentationSaveFilePath));
FIntPoint DestSize(ActualRenderTarget->SizeX, ActualRenderTarget->SizeY);
TArray<uint8> ActualCompressedBitmap;
TArray<uint8> SegmentationCompressedBitmap;
FImageUtils::CompressImageArray(DestSize.X, DestSize.Y, ActualRawData, ActualCompressedBitmap);
FImageUtils::CompressImageArray(DestSize.X, DestSize.Y, SegmentationRawData, SegmentationCompressedBitmap);
FFileHelper::SaveArrayToFile(ActualCompressedBitmap, *ActualSaveFilePath);
FFileHelper::SaveArrayToFile(SegmentationCompressedBitmap, *SegmentationSaveFilePath);
}