I’m capturing depth and normal image, then reconstructing them into the points and normals, but there is a offset in the point, and normals (offset in case of uneven landscape, fine with flat landscape). Im using custom shaders, captring Scenetexture:WorldDepth and Scenetexture:WorldNormals into rendertargets. and this is my depth to points reconstruction function,
// params for converting depth into world cordinates
float AspectRatio=result->imageWidth/(float)result->imageHeight;
FMinimalViewInfo ViewInfo;
FMatrix ProjectionMatrix;
params->depthCaptures[idx]->GetCameraView(0.0f,ViewInfo);
ProjectionMatrix=AdjustProjectionMatrixForRHI(ViewInfo.CalculateProjectionMatrix());
FMatrix InvProjMatrix=ProjectionMatrix.Inverse();
FMatrix InvViewMatrix = FMatrix(
FPlane(0, 1, 0, 0),
FPlane(0, 0, 1, 0),
FPlane(1, 0, 0, 0),
FPlane(0, 0, 0, 1)) * FRotationTranslationMatrix(ViewInfo.Rotation,ViewInfo.Location);
FVector captureForward=params->depthCaptures[idx]->GetForwardVector();
//converting to world cordinates and image data processing
for (int y=0;y<result->imageHeight;y++)
{
for (int x=0;x<result->imageWidth;x++)
{
int Index=y*result->imageWidth+x;
float depth=result->worldDepth[Index];
// depth to world cordinates
if(depth>0.99*far)
{
result->worldPoints[Index]=FVector(NAN,NAN,NAN);
continue;
}
float U= (float)(x+0.5)/result->imageWidth; // adding 0.5 to move from the corner of the pixel to the centre
float V= (float)(y+0.5)/result->imageHeight; // adding 0.5 to move from the corner of the pixel to the centre
float ScreenX= (U-0.5f)*2.0f;
float ScreenY= (1.0f-V-0.5f)*2.0f;
FVector4 rayStartProjectionSpace=FVector4(ScreenX,ScreenY,1.0f,1.0f);
FVector4 rayEndProjectionSpace=FVector4(ScreenX,ScreenY,0.01f,1.0f);
rayStartProjectionSpace =InvProjMatrix.TransformFVector4(rayStartProjectionSpace);
rayEndProjectionSpace=InvProjMatrix.TransformFVector4(rayEndProjectionSpace);
FVector rayStart = FVector(rayStartProjectionSpace.X,rayStartProjectionSpace.Y,rayStartProjectionSpace.Z);
FVector rayEnd = FVector(rayEndProjectionSpace.X,rayEndProjectionSpace.Y,rayEndProjectionSpace.Z);
rayStart/=rayStartProjectionSpace.W;
rayEnd/=rayEndProjectionSpace.W;
FVector rayDir=rayEnd-rayStart;
rayDir=rayDir.GetSafeNormal();
FVector worldPos= ViewInfo.Location; //InvViewMatrix.TransformPosition(rayStart);
rayDir=InvViewMatrix.TransformVector(rayDir).GetSafeNormal();
rayDir.Normalize();
depth=depth/(FVector::DotProduct(rayDir,captureForward)); // depth/cos(theta) // to offset direction depth
worldPos+=rayDir*depth;
result->worldPoints[Index]=worldPos;
}
}
there is anywhere from 3-20 deg of difference in normals, and the difference in the points scales with how far the point is from the capture component/ depth.
blue is the raycasting point and impactnormal.
yellow and red is reconstructed point (depth) and normal.
