Primitive disappears when sampling texture in custom global shader

Development Rendering
, , ,
1

Hi, I’m using a post opaque delegate to do a custom rendering pass where I create some buffers and textures and draw to the screen using the IRendererModule->RegisterPostOpaqueRenderDelegate function.

I’m using Ubuntu 22.04, Unreal Engine 5.5, using Vulkan.

What I am trying to do is to fill a texture with a font atlas, generate quads for each char of text, and then pass the atlas texture and the char data to a shader to render the chars to quads on the screen. The quad rendering, minus the atlas texture, is correct and working as expected, they get drawn on the screen in the right position, size, and colour. The only issue I’m having is the following:

When I run the code, without sampling the texture, the quads render fine with just their base colour, but when I use the Text variable here:

float4 MainPS(VertexOutput Input): SV_Target
{
	float Text = AtlasTexture.SampleLevel(AtlasSampler, Input.TexCoord, 0).x;
	
	float4 Output;
	Output.xyz = Input.CharColour;
	Output.w = 1; // Output.w = Text; // <------------ this is what causes the quads to disappear
	return Output;
}

Then the quads disappear, without error or warning or any other kind of output.

I have verified the following in nvidia Nsight:

  1. The texture is filled with valid data and bound at the time of drawing
  2. The vertex buffer, index buffer, and chars buffer are filled with valid data and bound at the time of rendering
  3. All vertex and pixel shader parameters are set and filled with valid data
  4. The correct number of indices, primitives, and instances is being passed to RHICommandList.DrawIndexedPrimitive

I have been through many pages of the engine code looking for examples and existing code that I can use to figure out if I’m using the APIs correctly.

I have been through every line of code and verified that it should be correct, and been through every tutorial and forum post that I can find that is even vaguely related to this issue, but I haven’t found any solution or information that I can use to solve the problem.

Below I will list the full code, any help or information would be appreciated!

#include "TextRenderer.h"
#include "CommonRenderResources.h"
#include "RenderGraphBuilder.h"
#include "Interfaces/IPluginManager.h"
#include "SceneTextures.h"
#include "SceneView.h"
#include "Kismet/GameplayStatics.h"
#include "GenerateMips.h"

#define STB_TRUETYPE_IMPLEMENTATION
#include "TextRenderer/External/stb_truetype.h"

#define LOCTEXT_NAMESPACE "FTextRendererModule"

struct FMagmaText
{
	FString Text;
	FVector3f WorldPosition;
	FString Font;
	FVector3f Colour;
	float Size;
};

struct FCharAtlas
{
	TRefCountPtr<IPooledRenderTarget> ExternalTexture;
	stbtt_bakedchar *Chars = 0;
	uint8 *Pixels = 0;
	int Width, Height;
};

struct FCharData
{
	FVector4f TexCoords;
	FVector4f Quad;
	FVector3f Colour;
	float Pad0;
};

class FTextRendererVS : public FGlobalShader
{
	DECLARE_SHADER_TYPE(FTextRendererVS, Global, TEXTRENDERER_API)
	SHADER_USE_PARAMETER_STRUCT(FTextRendererVS, FGlobalShader)
	
	BEGIN_SHADER_PARAMETER_STRUCT(FParameters, )
		SHADER_PARAMETER_RDG_BUFFER_SRV(StructuredBuffer<FCharData>, Chars)
		SHADER_PARAMETER(FVector2f, TargetSize)
	END_SHADER_PARAMETER_STRUCT()
};

class FTextRendererPS : public FGlobalShader
{
public:
	DECLARE_SHADER_TYPE(FTextRendererPS, Global, TEXTRENDERER_API)
	SHADER_USE_PARAMETER_STRUCT(FTextRendererPS, FGlobalShader)

	BEGIN_SHADER_PARAMETER_STRUCT(FParameters, )
		SHADER_PARAMETER_RDG_TEXTURE(Texture2D, AtlasTexture)
		SHADER_PARAMETER_SAMPLER(SamplerState, AtlasSampler)
	END_SHADER_PARAMETER_STRUCT()
};

void FTextRendererModule::StartupModule()
{
	UE_LOG(LogTemp, Warning, TEXT("FTextRendererModule::StartupModule"));
	
	FString PluginShaderDir = FPaths::Combine(IPluginManager::Get().FindPlugin(TEXT("TextRenderer"))->GetBaseDir(), TEXT("Shaders"));
	AddShaderSourceDirectoryMapping(TEXT("/TextShaders"), PluginShaderDir);
	
	const FName RendererModuleName("Renderer");
	IRendererModule* RendererModule = FModuleManager::GetModulePtr<IRendererModule>(RendererModuleName);
	if (RendererModule)
	{
		OnPostOpaqueHandle = RendererModule->RegisterPostOpaqueRenderDelegate(FPostOpaqueRenderDelegate::CreateRaw(this, &FTextRendererModule::OnPostOpaque));
	}
}

void FTextRendererModule::ShutdownModule()
{
	UE_LOG(LogTemp, Warning, TEXT("FTextRendererModule::ShutdownModule"));
	
	const FName RendererModuleName("Renderer");
	IRendererModule* RendererModule = FModuleManager::GetModulePtr<IRendererModule>(RendererModuleName);
	if (RendererModule)
	{
		RendererModule->RemovePostOpaqueRenderDelegate(OnPostOpaqueHandle);
	}
}

struct FCharVertex
{
	FVector3f Position;
	FVector2f TexCoord;
};

class FCharVertexDeclaration : public FRenderResource
{
public:
	FVertexDeclarationRHIRef VertexDeclarationRHI;
	
	virtual void InitRHI(FRHICommandListBase& RHICmdList) override
	{
		FVertexDeclarationElementList Elements;
		
		Elements.Add(FVertexElement(0, STRUCT_OFFSET(FCharVertex, Position), VET_Float3, 0, sizeof(FCharVertex)));
		Elements.Add(FVertexElement(0, STRUCT_OFFSET(FCharVertex, TexCoord), VET_Float2, 1, sizeof(FCharVertex)));
		
		VertexDeclarationRHI = PipelineStateCache::GetOrCreateVertexDeclaration(Elements);
	}
	virtual void ReleaseRHI() override
	{
		VertexDeclarationRHI.SafeRelease();
	}
};

TGlobalResource<FCharVertexDeclaration> GCharVertexDeclaration;

BEGIN_SHADER_PARAMETER_STRUCT(FCopyTextureParameters, TEXTRENDERER_API)
	RDG_TEXTURE_ACCESS(Dest, ERHIAccess::CopyDest)
END_SHADER_PARAMETER_STRUCT()

BEGIN_SHADER_PARAMETER_STRUCT(FTextRendererParameters, TEXTRENDERER_API)
	SHADER_PARAMETER_STRUCT_INCLUDE(FTextRendererVS::FParameters, VertParameters)
	SHADER_PARAMETER_STRUCT_INCLUDE(FTextRendererPS::FParameters, PixelParameters)

	RDG_TEXTURE_ACCESS(AtlasTexture, ERHIAccess::SRVGraphicsPixel)
	RDG_BUFFER_ACCESS(CharBuffer, ERHIAccess::SRVGraphicsNonPixel)

	RDG_BUFFER_ACCESS(VertexBuffer, ERHIAccess::VertexOrIndexBuffer)
	RDG_BUFFER_ACCESS(IndexBuffer, ERHIAccess::VertexOrIndexBuffer)

	RENDER_TARGET_BINDING_SLOTS()
END_SHADER_PARAMETER_STRUCT()

void FTextRendererModule::OnPostOpaque(FPostOpaqueRenderParameters &Parameters)
{
	FRDGBuilder& GraphBuilder = *Parameters.GraphBuilder;
	
	struct FCharBatch
	{
		TArray<FCharData> Chars;
	};
	
	TMap<uint32, FCharBatch> Batches;
	
	for (FMagmaText &String: Strings)
	{		
		uint32 Hash = GetTypeHash(String.Font) ^ GetTypeHash(String.Size);
		if (!AtlasMap.Contains(Hash))
		{
			FString Path = FPaths::ProjectContentDir() / String.Font;

			UE_LOG(LogTemp, Warning, TEXT("Creating font atlas for font %s"), *Path);

			TArray64<uint8> Data;
			FFileHelper::LoadFileToArray(Data, *Path);
			
			FCharAtlas Atlas;
			Atlas.Width = 1024;
			Atlas.Height = 1024;
			Atlas.Pixels = (uint8 *)FMemory::Malloc(Atlas.Width * Atlas.Height);
			Atlas.Chars = (stbtt_bakedchar *)FMemory::Malloc(sizeof(stbtt_bakedchar) * 96);
			stbtt_BakeFontBitmap(Data.GetData(), stbtt_GetFontOffsetForIndex(Data.GetData(), 0), String.Size, Atlas.Pixels, Atlas.Width, Atlas.Height, 32, 96, Atlas.Chars);

			AtlasMap.Add(Hash, Atlas);
		}

		FVector2D ScreenPos;
		FSceneView::ProjectWorldToScreen(FVector(String.WorldPosition), ViewRect, ViewProjectionMatrix, ScreenPos);
		
		FVector2f Position = FVector2f(ScreenPos); 
			
		FCharBatch &Batch = Batches.FindOrAdd(Hash);
		FCharAtlas *Atlas = AtlasMap.Find(Hash);

		for (int i = 0; i < String.Text.Len(); i++)
		{
			char C = String.Text[i] - 32;

			stbtt_aligned_quad Quad;
			stbtt_GetBakedQuad(Atlas->Chars, Atlas->Width, Atlas->Height, C, &Position.X, &Position.Y, &Quad, 1);

			FCharData Data;
			Data.TexCoords = FVector4f(Quad.s0, Quad.t0, Quad.s1, Quad.t1);
			Data.Quad = FVector4f(Quad.x0, Quad.y0, Quad.x1, Quad.y1);
			Data.Colour = FVector3f(String.Colour);
			Batch.Chars.Add(Data);
		}
	}

	TArray<uint32> Keys;
	Batches.GenerateKeyArray(Keys);

	for (uint32 Key: Keys)
	{
		FCharAtlas *Atlas = AtlasMap.Find(Key);
		if (!Atlas->ExternalTexture)
		{
			FRDGTextureDesc Desc = FRDGTextureDesc::Create2D(FIntPoint(Atlas->Width, Atlas->Height), PF_R8, FClearValueBinding::Green, ETextureCreateFlags::ShaderResource | ETextureCreateFlags::RenderTargetable);
			Atlas->ExternalTexture = GraphBuilder.ConvertToExternalTexture(
				GraphBuilder.CreateTexture(Desc, TEXT("TextRenderer_AtlasTexture"))
			);
			
			FCopyTextureParameters *CopyParameters = GraphBuilder.AllocParameters<FCopyTextureParameters>();
			CopyParameters->Dest = GraphBuilder.RegisterExternalTexture(Atlas->ExternalTexture);
			
			GraphBuilder.AddPass(RDG_EVENT_NAME("TextRenderer_UploadAtlas"), CopyParameters, ERDGPassFlags::NeverCull | ERDGPassFlags::Copy, [Atlas, CopyParameters](FRHICommandListImmediate &RHICommandList)
			{
				RHICommandList.UpdateTexture2D(CopyParameters->Dest->GetRHI(), 0, FUpdateTextureRegion2D(0, 0, 0, 0, Atlas->Width, Atlas->Height), Atlas->Width, Atlas->Pixels);
			});

			FGenerateMips::Execute(GraphBuilder, GMaxRHIFeatureLevel, GraphBuilder.RegisterExternalTexture(Atlas->ExternalTexture));
		}
	}
	
	for (uint32 Key: Keys)
	{
		FTextRendererParameters *TextParameters = GraphBuilder.AllocParameters<FTextRendererParameters>();
		TextParameters->RenderTargets[0] = FRenderTargetBinding(Parameters.ColorTexture, ERenderTargetLoadAction::ELoad);
		TextParameters->RenderTargets.DepthStencil = FDepthStencilBinding(Parameters.DepthTexture, ERenderTargetLoadAction::ELoad, ERenderTargetLoadAction::ENoAction, FExclusiveDepthStencil::DepthWrite_StencilNop);
			
		if (!ExternalVertexBuffer)
		{
			FCharVertex Verts[4];
			Verts[0] = (FCharVertex) {FVector3f(0, 0, 0), FVector2f(0, 0)};
			Verts[1] = (FCharVertex) {FVector3f(1, 0, 0), FVector2f(1, 0)};
			Verts[2] = (FCharVertex) {FVector3f(1, 1, 0), FVector2f(1, 1)};
			Verts[3] = (FCharVertex) {FVector3f(0, 1, 0), FVector2f(0, 1)};
			FRDGBufferRef VertexBuffer = CreateVertexBuffer(GraphBuilder, TEXT("TextRenderer_VertexBuffer"), FRDGBufferDesc::CreateBufferDesc(sizeof(FCharVertex), 4), Verts, sizeof(Verts));
			ExternalVertexBuffer = GraphBuilder.ConvertToExternalBuffer(VertexBuffer);
		}
		
		TextParameters->VertexBuffer = GraphBuilder.RegisterExternalBuffer(ExternalVertexBuffer);

		if (!ExternalIndexBuffer)
		{
			uint32 Indices[6] = {
				0, 1, 2, 2, 3, 0
			};
			FRDGBufferRef IndexBuffer = CreateVertexBuffer(GraphBuilder, TEXT("TextRenderer_IndexBuffer"), FRDGBufferDesc::CreateBufferDesc(sizeof(uint32), 6), Indices, sizeof(Indices));
			ExternalIndexBuffer = GraphBuilder.ConvertToExternalBuffer(IndexBuffer);
		}
		
		TextParameters->IndexBuffer = GraphBuilder.RegisterExternalBuffer(ExternalIndexBuffer);

		FCharBatch *Batch = Batches.Find(Key);
		FRDGBufferRef CharBuffer = CreateStructuredBuffer(GraphBuilder, TEXT("TextRenderer_CharBuffer"), sizeof(FCharData), Batch->Chars.Num(), Batch->Chars.GetData(), Batch->Chars.NumBytes());
		TextParameters->CharBuffer = CharBuffer;

		FCharAtlas *Atlas = AtlasMap.Find(Key);
		TextParameters->AtlasTexture = GraphBuilder.RegisterExternalTexture(Atlas->ExternalTexture);

		FVector2f TargetSize = FVector2f(Parameters.ColorTexture->Desc.Extent);

		TextParameters->VertParameters.Chars = GraphBuilder.CreateSRV(CharBuffer);
		TextParameters->VertParameters.TargetSize = TargetSize;
		
		TextParameters->PixelParameters.AtlasTexture = TextParameters->AtlasTexture; 
		TextParameters->PixelParameters.AtlasSampler = TStaticSamplerState<SF_Bilinear>::GetRHI();
		
		int CharCount = Batch->Chars.Num();
		GraphBuilder.AddPass(RDG_EVENT_NAME("TextRenderer_RenderChars"), TextParameters, ERDGPassFlags::Raster,
			[this, Atlas, TargetSize, TextParameters, CharCount](FRHICommandListImmediate &RHICommandList)
		{
			FGlobalShaderMap *ShaderMap = GetGlobalShaderMap(GMaxRHIFeatureLevel);
			TShaderMapRef<FTextRendererVS> VertexShader(ShaderMap);
			TShaderMapRef<FTextRendererPS> PixelShader(ShaderMap);

			FGraphicsPipelineStateInitializer GraphicsPSOInit;
			RHICommandList.ApplyCachedRenderTargets(GraphicsPSOInit);
			GraphicsPSOInit.BlendState = TStaticBlendState<>::GetRHI();
			GraphicsPSOInit.RasterizerState = TStaticRasterizerState<>::GetRHI();
			GraphicsPSOInit.DepthStencilState = TStaticDepthStencilState<false, CF_Always>::GetRHI();
			GraphicsPSOInit.BoundShaderState.VertexDeclarationRHI = GCharVertexDeclaration.VertexDeclarationRHI;
			GraphicsPSOInit.BoundShaderState.VertexShaderRHI = VertexShader.GetVertexShader();
			GraphicsPSOInit.BoundShaderState.PixelShaderRHI = PixelShader.GetPixelShader();
			GraphicsPSOInit.PrimitiveType = PT_TriangleList;
			SetGraphicsPipelineState(RHICommandList, GraphicsPSOInit, 0);
	
			FRHIBatchedShaderParameters &BatchedParameters = RHICommandList.GetScratchShaderParameters();
				
			SetShaderParameters(BatchedParameters, VertexShader, TextParameters->VertParameters);
			SetShaderParameters(BatchedParameters, PixelShader, TextParameters->PixelParameters);
			
			RHICommandList.SetBatchedShaderParameters(VertexShader.GetVertexShader(), BatchedParameters);
			RHICommandList.SetBatchedShaderParameters(PixelShader.GetVertexShader(), BatchedParameters);
				
			RHICommandList.SetStreamSource(0, TextParameters->VertexBuffer->GetRHI(), 0);
			RHICommandList.DrawIndexedPrimitive(TextParameters->IndexBuffer->GetRHI(), 0, 0, 4, 0, 2, CharCount);
		});
	}
}

void FTextRendererModule::AddText(TArray<FMagmaText> Strings)
{
	this->Strings = Strings;
}

void FTextRendererModule::ClearText()
{
	Strings.Empty();
}

IMPLEMENT_SHADER_TYPE(, FTextRendererVS, TEXT("/TextShaders/TextRendererShader.usf"), TEXT("MainVS"), SF_Vertex);
IMPLEMENT_SHADER_TYPE(, FTextRendererPS, TEXT("/TextShaders/TextRendererShader.usf"), TEXT("MainPS"), SF_Pixel);

DECLARE_GPU_STAT_NAMED(Plugin_TextRenderer, TEXT("TextRenderer: Render Pixel Shader"));

#undef LOCTEXT_NAMESPACE
	
IMPLEMENT_MODULE(FTextRendererModule, TextRenderer);

Shader code:

#include "/Engine/Public/Platform.ush"
#include "/Engine/Private/Common.ush"

struct FCharData
{
	float4 TexCoord;
	float4 Quad;
	float3 Colour;
	float Pad0;
};

StructuredBuffer<FCharData> Chars;
float2 TargetSize;

struct VertexOutput
{
	float4 ClipPosition: SV_POSITION;
	float3 CharColour: TEXCOORD0;
	float2 TexCoord: TEXCOORD1;
};

VertexOutput MainVS(in float3 InPosition: ATTRIBUTE0, in float2 InTexCoord: ATTRIBUTE1, in int InstanceId: SV_InstanceId)
{
	FCharData Char = Chars[InstanceId];
	
	float2 Position = Char.Quad.xy + (Char.Quad.zw - Char.Quad.xy)*InTexCoord;
	float2 TexCoord = Char.TexCoord.xy + (Char.TexCoord.zw - Char.TexCoord.xy)*InTexCoord;

	Position = -1.0 + 2.0 * ((InPosition.xy + Position) / TargetSize);
	Position *= float2(1.0, -1.0);
	
	VertexOutput Output;
	Output.ClipPosition = float4(Position, 0.0, 1.0);
	Output.CharColour = Char.Colour;
	Output.TexCoord = TexCoord;
	return Output;
}

Texture2D AtlasTexture;
SamplerState AtlasSampler;

float4 MainPS(VertexOutput Input): SV_Target
{
	float Text = AtlasTexture.SampleLevel(AtlasSampler, Input.TexCoord, 0).x;
	
	float4 Output;
	Output.xyz = Input.CharColour;
	Output.w = 1;
	return Output;
}