Depth map from scene capture2d

Development Programming & Scripting
, , ,
1

I want to create a 16bit depth map from my scene using the SceneCapture2d component. Then I want to save this 16bit depth map as a png file. To some extent I was successful, but in other way no.
The problem here is that, the generated png shows 4 times the scene in a row. Also most parts of the generated image is also black.

Here I attach the generated png file.

depth_0002
depth_0002640×480 12.8 KB

The scene capture 2d, is looking at a static mesh. But in the generated image I see four meshes.

2

In the SceneCapture2D I assigned these values:
Capture Source= ‘SceneDepth in R’,
Texture Target= ‘NewTextureRenderTarget2D’,

Also in NewTextureRenderTarget2D I assigned these values:
Render Target Format = ‘RTF R16f’.

3

Here I bring the two functions that I use for saving the png file.
SaveTo16 and CaptureSkeletalMeshDepth. Using both, there are four times of the char repeated.

// Fill out your copyright notice in the Description page of Project Settings.
#include "ImageUtils.h"
#include "Engine/TextureRenderTarget2D.h"

#include "IImageWrapper.h"
#include "IImageWrapperModule.h"
#include "Modules/ModuleManager.h"

#include "Misc/FileHelper.h"
#include "Misc/Paths.h"
#include "Components/SceneCaptureComponent2D.h"
#include "Components/SkeletalMeshComponent.h"
#include "Engine/TextureRenderTarget2D.h"
#include "UObject/UObjectIterator.h"

#include "SavingLibrary.h"




bool USavingLibrary::SaveTo16(UTextureRenderTarget2D* RenderTarget, const FString& FilePath)
{
    if (!RenderTarget)
    {
        UE_LOG(LogTemp, Error, TEXT("SaveTo16: RenderTarget is null."));
        return false;
    }

    if (RenderTarget->RenderTargetFormat != RTF_R16f)
    {
        UE_LOG(LogTemp, Warning, TEXT("SaveTo16: Expected RTF_R16f, got a different format. Continuing anyway."));
    }

    FTextureRenderTargetResource* RTResource = RenderTarget->GameThread_GetRenderTargetResource();
    if (!RTResource)
    {
        UE_LOG(LogTemp, Error, TEXT("SaveTo16: Could not get render target resource."));
        return false;
    }

    const int32 Width  = RenderTarget->SizeX;
    const int32 Height = RenderTarget->SizeY;

    // Read back as FFloat16Color — works for single-channel R16F targets too;
    // only the R channel (our SceneDepth) will carry meaningful data.
    TArray<FFloat16Color> RawData;
    if (!RTResource->ReadFloat16Pixels(RawData))
    {
        UE_LOG(LogTemp, Error, TEXT("SaveTo16: ReadFloat16Pixels failed."));
        return false;
    }
        UE_LOG(LogTemp, Warning, TEXT("First pixel: R=%f G=%f B=%f A=%f"),
            RawData[0].R.GetFloat(),
            RawData[0].G.GetFloat(),
            RawData[0].B.GetFloat(),
            RawData[0].A.GetFloat());
    if (RawData.Num() != Width * Height)
    {
        UE_LOG(LogTemp, Error, TEXT("SaveTo16: Pixel count (%d) doesn't match target size (%d)."),
            RawData.Num(), Width * Height);
        return false;
    }

    // Bit-copy each half-float's raw 16-bit pattern into a uint16 buffer.
    // No normalization/clamping — this preserves the exact raw SceneDepth
    // value and allows exact reconstruction later via FFloat16(EncodedValue).
    TArray<uint16> GrayPixels;
    GrayPixels.SetNumUninitialized(Width * Height);
    for (int32 i = 0; i < RawData.Num(); ++i)
    {
        GrayPixels[i] = RawData[i].R.Encoded;
    }

    IImageWrapperModule& ImageWrapperModule =
        FModuleManager::LoadModuleChecked<IImageWrapperModule>(FName("ImageWrapper"));
    TSharedPtr<IImageWrapper> ImageWrapper = ImageWrapperModule.CreateImageWrapper(EImageFormat::PNG);
    if (!ImageWrapper.IsValid())
    {
        UE_LOG(LogTemp, Error, TEXT("SaveTo16: Failed to create PNG image wrapper."));
        return false;
    }

    const int64 RawSizeBytes = static_cast<int64>(GrayPixels.Num()) * sizeof(uint16);
    if (!ImageWrapper->SetRaw(GrayPixels.GetData(), RawSizeBytes, Width, Height, ERGBFormat::Gray, 16))
    {
        UE_LOG(LogTemp, Error, TEXT("SaveTo16: SetRaw failed."));
        return false;
    }

    const TArray64<uint8> PNGData = ImageWrapper->GetCompressed(100);
    if (PNGData.Num() == 0)
    {
        UE_LOG(LogTemp, Error, TEXT("SaveTo16: PNG compression produced no data."));
        return false;
    }

    if (!FFileHelper::SaveArrayToFile(PNGData, *FilePath))
    {
        UE_LOG(LogTemp, Error, TEXT("SaveTo16: Failed to write file to '%s'."), *FilePath);
        return false;
    }

    return true;
}


bool USavingLibrary::CaptureSkeletalMeshDepth(USceneCaptureComponent2D* SceneCapture, const FString& FilePath, float MaxDistance)
{
    if (!SceneCapture)
    {
        UE_LOG(LogTemp, Error, TEXT("CaptureSkeletalMeshDepth: SceneCapture is null."));
        return false;
    }

    UWorld* World = SceneCapture->GetWorld();
    if (!World)
    {
        UE_LOG(LogTemp, Error, TEXT("CaptureSkeletalMeshDepth: No world context."));
        return false;
    }

    UTextureRenderTarget2D* RenderTarget = SceneCapture->TextureTarget;
    
    if (!RenderTarget)
    {
        UE_LOG(LogTemp, Error, TEXT("CaptureSkeletalMeshDepth: SceneCapture has no TextureTarget assigned."));
        return false;
    }
    
    const FVector CaptureLocation = SceneCapture->GetComponentLocation();
    const FVector CaptureForward  = SceneCapture->GetForwardVector();

    // Match the capture's actual field of view rather than a hardcoded cone.
    // Add a little slack (+10 deg) so meshes with bulk near the frustum edge
    // aren't excluded just because their pivot/origin is slightly outside it.
    const float HalfFovDeg = (SceneCapture->FOVAngle * 0.5f) + 10.f;
    const float CosThreshold = FMath::Cos(FMath::DegreesToRadians(HalfFovDeg));

    SceneCapture->PrimitiveRenderMode = ESceneCapturePrimitiveRenderMode::PRM_UseShowOnlyList;
    SceneCapture->ShowOnlyComponents.Empty();
    SceneCapture->ShowOnlyActors.Empty();
    SceneCapture->bCaptureEveryFrame = false;
    SceneCapture->bCaptureOnMovement = false;
    int32 NumAdded = 0;

    for (TObjectIterator<USkeletalMeshComponent> It; It; ++It)
    {
        USkeletalMeshComponent* SkelComp = *It;
        if (!IsValid(SkelComp) || SkelComp->GetWorld() != World || !SkelComp->IsVisible())
        {
            continue;
        }

        FVector ToComponent = SkelComp->GetComponentLocation() - CaptureLocation;
        const float Distance = ToComponent.Size();
        if (Distance < KINDA_SMALL_NUMBER || Distance > MaxDistance)
        {
            continue;
        }
        ToComponent /= Distance;

        const float Dot = FVector::DotProduct(CaptureForward, ToComponent);
        if (Dot >= CosThreshold)
        {
            SceneCapture->ShowOnlyComponents.Add(SkelComp);
            ++NumAdded;
        }
    }

    if (NumAdded == 0)
    {
        UE_LOG(LogTemp, Warning, TEXT("CaptureSkeletalMeshDepth: No skeletal meshes found in front of the capture — depth map will be empty."));
    }

    // Force an immediate capture so the show-only list takes effect right now,
    // regardless of bCaptureEveryFrame's setting.
    SceneCapture->CaptureScene();

    return SaveTo16(RenderTarget, FilePath);
}
4

Here is what the scene is really looking like.
Please help me how I can get rid of four scene repetition issue.
Also as you see on the first image, more than half of the image is empty, can you help me also to fix that?

MyScreenshot
MyScreenshot1201×830 172 KB