Dynamically Spawn SceneCapture Visible in HUD

Development Programming & Scripting UI
, , , ,
1

I have a WheeledVehicle which I want to be able to attach ASceneCapture2D. When the vehicle is driven around during game play the output texture of the scene capture should be visible in the HUD. I planned to do this by creating a custom WrapBox widget which contains UImages of each SceneCapture. Bellow is the code

Vehicle Class

UENUM()
enum SensorType { RADAR = 0, LIDAR };

UCLASS(config = Game)
class USensorPlacementConfigData : public UObject
{
	GENERATED_BODY()
public:
	UPROPERTY(Config, BlueprintReadWrite, EditAnywhere)
		float rot_x;
	UPROPERTY(Config, BlueprintReadWrite, EditAnywhere)
		float rot_y;
	UPROPERTY(Config, BlueprintReadWrite, EditAnywhere)
		float rot_z;
	UPROPERTY(Config, BlueprintReadWrite, EditAnywhere)
		float rot_w;

	UPROPERTY(Config, BlueprintReadWrite, EditAnywhere)
		float scale_x;
	UPROPERTY(Config, BlueprintReadWrite, EditAnywhere)
		float scale_y;
	UPROPERTY(Config, BlueprintReadWrite, EditAnywhere)
		float scale_z;

	UPROPERTY(Config, BlueprintReadWrite, EditAnywhere)
		float trans_x;
	UPROPERTY(Config, BlueprintReadWrite, EditAnywhere)
		float trans_y;
	UPROPERTY(Config, BlueprintReadWrite, EditAnywhere)
		float trans_z;

	UPROPERTY(Config, BlueprintReadWrite, EditAnywhere)
		TEnumAsByte<SensorType> type;
};

UCLASS(Config = Game)
class WATOSIM_API ASensorMountedVehicle : public AWheeledVehicle
{
	GENERATED_BODY()
	

public:
	UPROPERTY(Config, BlueprintReadWrite, EditAnywhere)
		TArray<TSubclassOf<class USensorPlacementConfigData >> sensors;

	//UPROPERTY(BlueprintReadWrite, EditAnywhere)
	TArray<class ASceneCapture2D *> mountedSensors;

	//UPROPERTY(BlueprintReadWrite, EditAnywhere)
	TArray<class UTextureRenderTarget2D *> sensorRenderTargets;

public:
	virtual void BeginPlay() override;
	virtual void Tick(float DeltaTime) override;


	~ASensorMountedVehicle();
};

void ASensorMountedVehicle::BeginPlay()
{
	Super::BeginPlay();
	UE_LOG(LogTemp, Error, TEXT("NumSensor: %d"), sensors.Num());
	sensorRenderTargets.Reset();
	for (auto& sensor : sensors) {
		auto s = sensor.GetDefaultObject();
		UE_LOG(LogTemp, Error, TEXT("%f %f %f %f, %f %f %f, %f %f %f, %d"), 
			s->rot_x, s->rot_y, s->rot_z, s->rot_w, s->scale_x, s->scale_y, s->scale_z,
			s->trans_x, s->trans_y, s->trans_z, (int)s->type.GetValue()
		);

		//Make the render target for this sensor
		UTextureRenderTarget2D *renderTarget2D = NewObject<UTextureRenderTarget2D>();
		renderTarget2D->InitAutoFormat(512, 512);

		//Create the ScreenCaptureActure which needs to be spawned into the worl
		ASceneCapture2D *sceneCaptureActor = (class ASceneCapture2D *)GetWorld()->SpawnActor<ASceneCapture2D>(ASceneCapture2D::StaticClass());
		if (sceneCaptureActor) {
			//Make the transform for this sensor
			FTransform cameraTransform = FTransform(
				FQuat(s->rot_x, s->rot_y, s->rot_z, s->rot_w),
				FVector(s->scale_x, s->scale_y, s->scale_z),
				FVector(s->trans_x, s->trans_y, s->trans_z)
			);

			// Getting the actual component, registering and putting it where the camera was
			USceneCaptureComponent2D *sceneCaptureComponent = sceneCaptureActor->GetCaptureComponent2D();
			//sceneCaptureComponent->RegisterComponent();
			sceneCaptureComponent->SetRelativeTransform(cameraTransform);
			sceneCaptureComponent->FOVAngle = 90.0;
			sceneCaptureComponent->ProjectionType = ECameraProjectionMode::Perspective;
			
			// Setting the target
			sensorRenderTargets.Add(sceneCaptureComponent->TextureTarget);
			sceneCaptureComponent->TextureTarget = renderTarget2D;

			sceneCaptureComponent->bAutoActivate = true;
			sceneCaptureComponent->DetailMode = EDetailMode::DM_High;
			switch (sensor.GetDefaultObject()->type) {
			case SensorType::RADAR:
				UE_LOG(LogTemp, Error, TEXT("Found Radar"));
				sceneCaptureComponent->CaptureSource = SCS_SceneColorHDR;
				break;
			case SensorType::LIDAR:
				UE_LOG(LogTemp, Error, TEXT("Found Lidar"));
				sceneCaptureComponent->CaptureSource = SCS_SceneDepth;
				break;
			default:
				UE_LOG(LogTemp, Error, TEXT("Sensor config has unknown type"));
				break;
			}

			//sceneCaptureActor->RegisterAllComponents();
			mountedSensors.Add(sceneCaptureActor);
		}
		else {
			UE_LOG(LogTemp, Error, TEXT("SceneCapture loaded is null"));
		}
	}
}

void ASensorMountedVehicle::Tick(float DeltaTime) {
	Super::Tick(DeltaTime);

	/*for (int sensNum = 0; sensNum < mountedSensors.Num(); sensNum++) {
		USceneCaptureComponent2D *sceneCaptureComponent = mountedSensors[sensNum]->GetCaptureComponent2D();
		sceneCaptureComponent->CaptureScene();

		// Telling the target to "refresh"
		UTextureRenderTarget2D *renderTarget2D = sensorRenderTargets[sensNum];
		renderTarget2D->UpdateResourceImmediate();

		// Creating the actual texture
		//UTexture2D *newTexture = renderTarget2D->ConstructTexture2D(this, FString("Snapshot"), EObjectFlags::RF_NoFlags);
	
		//use the texture for something fun...
	}*/
}

ASensorMountedVehicle::~ASensorMountedVehicle()
{
}

Custom Wrapper Class

UCLASS()
class WATOSIM_API USensorHUDWrapBox : public UWrapBox
{
	GENERATED_BODY()
	
		TArray<class UImage *> sensorsViews;

	class ASensorMountedVehicle *vehicle;

public:
	void InitSensorViews(class ASensorMountedVehicle *v);

	//UFUNCTION(BlueprintCallable)

	~USensorHUDWrapBox();
};

void USensorHUDWrapBox::InitSensorViews(ASensorMountedVehicle *v)
{
	UE_LOG(LogTemp, Error, TEXT("USensorHUDWrapBox"));
	vehicle = v;
	if (vehicle) {
		UE_LOG(LogTemp, Error, TEXT("Here 2"));
		for (int i = 0; i < vehicle->sensorRenderTargets.Num(); i++) {
			UE_LOG(LogTemp, Error, TEXT("Here 3"));
			class UImage *image = (class UImage *)NewObject<class UImage>(UImage::StaticClass());
			image->SetBrushFromTexture((class UTexture2D *)vehicle->sensorRenderTargets[i]);
			
			UCanvasPanelSlot* canvasSlot = Cast<UCanvasPanelSlot>(AddChild(image));

			sensorsViews.Add(image);
		}
	}
}

USensorHUDWrapBox::~USensorHUDWrapBox()
{
	for (int i = 0; i < sensorsViews.Num(); i++) {
		delete sensorsViews[i];
	}
}

Custom HUD Class

void AChevyBoltHUD::BeginPlay()
{
	Super::BeginPlay();

	//Establish the PC
	ThePC = GetOwningPlayerController();

	//How to get a ref to your custom PC
	//AYourPlayerController* YourChar = Cast<AYourPlayerController>(ThePC);

	//How to Get The Character
	vehicle = Cast<ASensorMountedVehicle>(GetOwningPawn());

	sensorView = (USensorHUDWrapBox *)NewObject<class USensorHUDWrapBox>();
	sensorView->InitSensorViews(vehicle);
	sensorView->AddToRoot();
}

The code runs fine and outputs the logs messages in the correct order. However, nothing shows up on the HUD. Any ideas? I know the code is being run and the AddChild() function is being called.

2

If your AChevyBoltHUD inherits from AHUD then you need using DrawHUD() instead BeginPlay().

void AMyHUD::BeginPlay()
{
	Super::BeginPlay();

	DrawText(TEXT("Welcome in BeginPlay"), FColor::White, 100, 90);
}

void AMyHUD::DrawHUD()
{
	Super::DrawHUD();

	DrawText(TEXT("DrawHUD here"), FColor::White, 100, 90);
}

Text in BeginPlay doesn’t show, but the text in DrawHUD is visible.

3

I’m not trying to draw text in begin play. What I was trying to do in the BeginPlay() was put a bunch of images in the HUD wrap box. I was hoping that these images would be tied to the sensor texture so that when the next capture happens the texture and image displayed in the HUD gets updated.

4

I understand that SceneCapture2D creates a texture and you want to display this texture on HUD as below

233674-scenecapture.gif

5

Yes! that looks like the exact effect I am aiming for. Do I have to continuously add the images to the HUDS’ Wrap box widget? I thought adding the image in the BeginPlay() would have it there during the Draw step automatically?

6

First off this:

sensorRenderTargets.Add(sceneCaptureComponent->TextureTarget);

do after (or better somewhere at the end)

sceneCaptureComponent->TextureTarget = renderTarget2D;

because

sensorRenderTargets.Add(UTextureRenderTarget2D * const & Item)

which means Item is constant pointer such a pointer cannot be moved.

I’m not using WrapBox. Only AChevyBoltHUD class:

void AChevyBoltHUD::BeginPlay()
{
	Super::BeginPlay();

	//How to Get The Character
	vehicle = Cast<ASensorMountedVehicle>(GetOwningPawn());
	
}

void AChevyBoltHUD::DrawHUD()
{
	Super::DrawHUD();
	
	if (vehicle)
	{
		//UTexture * HandleTexture = vehicle->sensorRenderTargets[0];
		// Declaration HandleTexture in ChevyBoltHUD.h:
		//    class UTexture * HandleTexture;

		if (HandleTexture)
		{
			DrawTexture(HandleTexture, 100.0f, 100.0f, 400.0f, 400.0f, 0.0f, 0.0f, 1.0f, 1.0f, FColor::White);
		}
	}
}

Only once you can set HandleTexture. I tested it quickly in ASensorMountedVehicle::Tick()

void ASensorMountedVehicle::Tick(float DeltaTime) 
{
	Super::Tick(DeltaTime);

	// only for a second set HandleTexture
	if(GetGameTimeSinceCreation() < 1.0f)
	{
		APlayerController * PlayerController = Cast<APlayerController>(Controller);
		AChevyBoltHUD * HUD = Cast<AChevyBoltHUD>(PlayerController->GetHUD());

		HUD->HandleTexture = sensorRenderTargets[0];
	}
}

Also make sure to select good sceneCaptureComponent->CaptureSource:
click for Video

7

I tried doing this

void AChevyBoltHUD::DrawHUD()
{
	Super::DrawHUD();
	for (int i = 0; i < vehicle->sensorRenderTargets.Num(); i++) {
		if (vehicle->sensorRenderTargets[i] != nullptr) {
			UE_LOG(LogTemp, Error, TEXT("Texture: %s"), vehicle->sensorRenderTargets[i]);
			DrawTexture(vehicle->sensorRenderTargets[i], 10.0f, 10.0f, 100.0f, 100.0f, 0.0f, 0.0f, 1.0f, 1.0f, FColor::White);
		}
	}
}

But i dont see any texture drawn on the sreen, I am using capture source “SCS_SceneColorHDR”

8

Also the log line looks like

LogTemp: Error: Texture: 꼠璯翻

9

You should use UE_LOG like this:

UE_LOG(LogTemp, Error, TEXT("Texture: %s"), *vehicle->sensorRenderTargets[i]->GetName());
10

If you sure, that you properly using SceneCapture and save textures, then you can try to manipulate CaptureSource and BlendMode in DrawTexture().
One of the parameters in DrawTexture() is EBlendMode (after FColor):

void AHUD::DrawTexture(
	UTexture* Texture, 
	float ScreenX, 
	float ScreenY, 
	float ScreenW, 
	float ScreenH, 
	float TextureU, 
	float TextureV, 
	float TextureUWidth, 
	float TextureVHeight, 
	FLinearColor Color, 
	EBlendMode BlendMode, //<--- HERE
	float Scale, 
	bool bScalePosition, 
	float Rotation, 
	FVector2D RotPivot
);

Below I have created a table that shows in which combination I am getting a texture on HUD.

234244-table.png
234244-table.png1698×471 12.3 KB

Also, you can change FColor to get another output.

If you want I can share my code (ChevyBoltHUD and SensorMountedVehicle class).

11

Sharing your code would be much appreciated, Will report back once I’ve tried your above suggestions

12

Work! thanks allot

13

The only thing slightly off is the SceneCaptureComponent doesnt follow the vehicle around. It just stays in the same spot it was spawned. Any idea how to attach it to the vehicle?

14

nvm fixed with sceneCaptureActor->AttachToActor(),thanks for the help!