Make FXAA work with custom tonemapper

Development Rendering
, , , ,
1

Hi,

So I wanted to get rid of unreal´s default tonemapper and made a postprocess material that replaces it. I already know this makes FXAA stop working so I am looking at the engine files to see if I can modify anything.

So far I have made it work when the antialiasing quality level enables FXAA_PC_CONSOLE (modified Fxaa3_11.ush); but since it looks meh to me I would like to make it work when on FXAA_PC, which I tried to do but got lost.

By the way, I don´t want to have to use FXAA_GREEN_AS_LUMA since my game shows pretty much just silhouettes of various colors.

Thanks,

fbgpro

2

Another note: I´ve already seen this post FXAA without tonemapper - Rendering - Epic Developer Community Forums but I stiil got nothing

3

I finally found out what texturegather() (inside Fxaa3_11.ush) was for, I´m trying to figure out how to use it if it helps me

4

Ok, I think I got something, I´m just going to test a version that works in linear colorspace before I post an answer

5

Here goes my answer:

Do all this inside Fxaa3_11.ush

Add this inside the if statement #if (FXAA_HLSL_5 == 1) that´s immediately above the GREEN AS LUMA OPTION SUPPORT FUNCTION:

//fbgpro adds more stuff here:
#define FxaaTexRed4(t, p) t.tex.GatherRed(t.smpl, clamp(p, t.UVMinMax.xy, t.UVMinMax.zw))
#define FxaaTexOffRed4(t, p, o) t.tex.GatherRed(t.smpl, clamp(p, t.UVMinMax.xy, t.UVMinMax.zw), o)
#define FxaaTexBlue4(t, p) t.tex.GatherBlue(t.smpl, clamp(p, t.UVMinMax.xy, t.UVMinMax.zw))
#define FxaaTexOffBlue4(t, p, o) t.tex.GatherBlue(t.smpl, clamp(p, t.UVMinMax.xy, t.UVMinMax.zw), o)

Then, go to the GREEN AS LUMA OPTION SUPPORT FUNCTION and make it look like this (maybe it does nothing for this problem but thats how mine looks like right now):

#if (FXAA_GREEN_AS_LUMA == 0)

	#if POST_PROCESS_ALPHA == 2
		// POST_PROCESS_ALPHA == 2 allows the tonemapper to not output Luma in alpha channel,
		// therefore need to compute it manually.
		FxaaFloat FxaaLuma(FxaaFloat4 rgba)
		{
			return sqrt(dot(rgba.rgb, FxaaFloat3(0.299f, 0.587f, 0.114f)));
		}

	#else
		//fbgpro copies the thing above
        //FxaaFloat FxaaLuma(FxaaFloat4 rgba) { return rgba.w; }
        FxaaFloat FxaaLuma(FxaaFloat4 rgba)
		{
			return sqrt(dot(rgba.rgb, FxaaFloat3(0.299f, 0.587f, 0.114f)));
		}

	#endif

#else
    FxaaFloat FxaaLuma(FxaaFloat4 rgba) { return rgba.y; }
#endif

Now go to the if statement after the long FxaaPixelShader function inside FXAA QUALITY - PC and replace that part with this:

FxaaFloat2 posM;
    posM.x = pos.x;
    posM.y = pos.y;
    #if (FXAA_GATHER4_ALPHA == 1) //fbgpro thinks this is when HLSL 5
        #if (FXAA_DISCARD == 0)
            FxaaFloat4 rgbyM = FxaaTexTop(tex, posM);
            rgbyM.w = sqrt(dot(rgbyM.xyz, half3(0.299f, 0.587f, 0.114f))); //fbgpro adds this line just in case
            #if (FXAA_GREEN_AS_LUMA == 0)
                #define lumaM sqrt(dot(rgbyM.xyz, half3(0.299f, 0.587f, 0.114f))) //fbgpro changed the w  
            #else
                #define lumaM rgbyM.y
            #endif
        #endif
        #if (FXAA_GREEN_AS_LUMA == 0)
            //fbgpro adds more fxaafloat4s
            FxaaFloat4 lumaRa = FxaaTexRed4(tex, posM);
            FxaaFloat4 lumaGa = FxaaTexGreen4(tex, posM);
            FxaaFloat4 lumaBa = FxaaTexBlue4(tex, posM);

            FxaaFloat4 lumaRb = FxaaTexOffRed4(tex, posM, FxaaInt2(-1, -1));
            FxaaFloat4 lumaGb = FxaaTexOffGreen4(tex, posM, FxaaInt2(-1, -1));
            FxaaFloat4 lumaBb = FxaaTexOffBlue4(tex, posM, FxaaInt2(-1, -1));

            FxaaFloat4 luma4A;
            FxaaFloat4 luma4B;
            //FxaaFloat4 luma4A = FxaaTexAlpha4(tex, posM);
            //FxaaFloat4 luma4B = FxaaTexOffAlpha4(tex, posM, FxaaInt2(-1, -1));
        #else
            FxaaFloat4 luma4A = FxaaTexGreen4(tex, posM);
            FxaaFloat4 luma4B = FxaaTexOffGreen4(tex, posM, FxaaInt2(-1, -1));
        #endif
        #if (FXAA_DISCARD == 1)
            #define lumaM sqrt(dot(half3(lumaRa.w,lumaGa.w,lumaBa.w), half3(0.299f, 0.587f, 0.114f))) //fbgpro replaces luma4A.w
        #endif

        //fbgpro adds eight lines
        luma4A.w = sqrt(dot(half3(lumaRa.w,lumaGa.w,lumaBa.w), half3(0.299f, 0.587f, 0.114f)));
        luma4A.x = sqrt(dot(half3(lumaRa.x,lumaGa.x,lumaBa.x), half3(0.299f, 0.587f, 0.114f)));
        luma4A.y = sqrt(dot(half3(lumaRa.y,lumaGa.y,lumaBa.y), half3(0.299f, 0.587f, 0.114f)));
        luma4A.z = sqrt(dot(half3(lumaRa.z,lumaGa.z,lumaBa.z), half3(0.299f, 0.587f, 0.114f)));

        luma4B.w = sqrt(dot(half3(lumaRb.w,lumaGb.w,lumaBb.w), half3(0.299f, 0.587f, 0.114f)));
        luma4B.x = sqrt(dot(half3(lumaRb.x,lumaGb.x,lumaBb.x), half3(0.299f, 0.587f, 0.114f)));
        luma4B.y = sqrt(dot(half3(lumaRb.y,lumaGb.y,lumaBb.y), half3(0.299f, 0.587f, 0.114f)));
        luma4B.z = sqrt(dot(half3(lumaRb.z,lumaGb.z,lumaBb.z), half3(0.299f, 0.587f, 0.114f)));

        #define lumaE luma4A.z
        #define lumaS luma4A.x
        #define lumaSE luma4A.y
        #define lumaNW luma4B.w
        #define lumaN luma4B.z
        #define lumaW luma4B.x
    #else
        FxaaFloat4 rgbyM = FxaaTexTop(tex, posM);
         rgbyM.w = sqrt(dot(rgbyM.xyz, half3(0.299f, 0.587f, 0.114f))); //fbgpro adds this line just in case
        #if (FXAA_GREEN_AS_LUMA == 0)
              #define lumaM sqrt(dot(rgbyM.xyz, half3(0.299f, 0.587f, 0.114f))) //fbgpro changed the w 
        #else
            #define lumaM rgbyM.y
        #endif
        FxaaFloat lumaS = FxaaLuma(FxaaTexOff(tex, posM, FxaaInt2( 0, 1), fxaaQualityRcpFrame.xy));
        FxaaFloat lumaE = FxaaLuma(FxaaTexOff(tex, posM, FxaaInt2( 1, 0), fxaaQualityRcpFrame.xy));
        FxaaFloat lumaN = FxaaLuma(FxaaTexOff(tex, posM, FxaaInt2( 0,-1), fxaaQualityRcpFrame.xy));
        FxaaFloat lumaW = FxaaLuma(FxaaTexOff(tex, posM, FxaaInt2(-1, 0), fxaaQualityRcpFrame.xy));
        
    #endif

Done. This should work with AA quality levels that use FXAA_PC (look at FXAAShader.usf inside the engine files), which would be high, epic and cinematic i think