StaticMeshComponent Add Indirect Capsule Shadow

Nice to meet you. I apologize if I’m not using this correctly. I’m trying to create a derived class of StaticMeshComponent and add a CapsuleShadow. I’ve managed to confirm that GetShadowShapes is called, but while I can see the DirectShadow, no matter what I do, the IndirectShadow isn’t reflected, which is causing some trouble.

The requirements allow for making the DirectShadow more faint or even just having a round shadow, but I would prefer not to alter the engine’s source code too much. I understand there may be some incorrect flag settings or configurations, but I’m having a hard time finding a combination that works effectively. If there are any suggestions or methods available, I’d appreciate if you could share them.

Although I am open to other methods, the solution needs to work with both forward and deferred rendering paths, which adds to the complexity of the situation.

class FAOStaticMeshSceneProxy; 

UENUM(BlueprintType)
enum class EPrimitiveAOType : uint8
{
	CIRCLE,
	CAPSULE
};


USTRUCT(BlueprintType)
struct MYSYSTEM_API FPrimitiveAO
{
	GENERATED_BODY()

	UPROPERTY(EditAnywhere, BlueprintReadWrite)
	EPrimitiveAOType Type = EPrimitiveAOType::CIRCLE;
	UPROPERTY(EditAnywhere, BlueprintReadWrite)
	FVector Center = FVector::ZeroVector;
	UPROPERTY(EditAnywhere, BlueprintReadWrite)
	FVector Rotation = FVector::ZeroVector;
	UPROPERTY(EditAnywhere, BlueprintReadWrite)
	float Length = 0.0f;
	UPROPERTY(EditAnywhere, BlueprintReadWrite)
	float Radius = 0.0f;

};

UCLASS(Blueprintable, ClassGroup = (Rendering, Common), hidecategories = (Object, Activation, "Components|Activation"), ShowCategories = (Mobility), editinlinenew, meta = (BlueprintSpawnableComponent))
class MYSYSTEM_API UAOStaticMeshComponent : public UStaticMeshComponent
{
	GENERATED_UCLASS_BODY()

public:

	~UAOStaticMeshComponent();

	UPROPERTY(EditAnywhere, BlueprintReadWrite, meta = (ToolTip = "Prim Shadow"))
	TArray<FPrimitiveAO> PrimitiveAO;

	virtual bool GetShadowIndirectOnly() const override
	{
		return true;
	}

	virtual FPrimitiveSceneProxy* CreateSceneProxy() override;

	virtual void CreateRenderState_Concurrent(FRegisterComponentContext* Context) override;

};

class MYSYSTEM_API FAOStaticMeshSceneProxy: public FStaticMeshSceneProxy
{
public:
	/**
	 * Constructor.
	 * @param	Component - mesh primitive being added
	 */
	FAOStaticMeshSceneProxy(const UAOStaticMeshComponent* Component);

	virtual ~FAOStaticMeshSceneProxy();

	virtual void GetShadowShapes(FVector PreViewTranslation, TArray<FCapsuleShape3f>& OutCapsuleShapes) const override;

	virtual bool HasDynamicIndirectShadowCasterRepresentation() const override;

	virtual bool CanBeOccluded() const override;

	virtual bool IsUsingDistanceCullFade() const override;

	virtual bool HasDistanceFieldRepresentation() const override;

	const UAOStaticMeshComponent* AOStaticMeshComponent = nullptr;

	virtual SIZE_T GetTypeHash() const override;

	virtual uint32 GetMemoryFootprint(void) const override;

	bool isAttacgGroup = false;
};

// cpp
UAOStaticMeshComponent::UAOStaticMeshComponent(const FObjectInitializer& ObjectInitializer)
	: UStaticMeshComponent(ObjectInitializer)
{

	bTickInEditor = true;

	PrimaryComponentTick.bCanEverTick = true;

	SetLightingChannels(true, false, false);
}

UAOStaticMeshComponent::~UAOStaticMeshComponent()
{
}


void UAOStaticMeshComponent::CreateRenderState_Concurrent(FRegisterComponentContext* Context)
{
	UStaticMeshComponent::CreateRenderState_Concurrent(Context);
}

FPrimitiveSceneProxy* UAOStaticMeshComponent::CreateSceneProxy()
{
	ERHIFeatureLevel::Type SceneFeatureLevel = GetWorld()->FeatureLevel;
	FAOStaticMeshSceneProxy* Result = ::new FAOStaticMeshSceneProxy(this);
	SceneProxy = Result;
	return Result;
}

/**
 * Constructor.
 * @param	Component - skeletal mesh primitive being added
 */
FAOStaticMeshSceneProxy::FAOStaticMeshSceneProxy(const UAOStaticMeshComponent* Component)
	: FStaticMeshSceneProxy((UStaticMeshComponent*)(Component), true)
{
	AOStaticMeshComponent = Component;

	bNeedsUnbuiltPreviewLighting = true;
//	bStaticLighting = true;
	bCastDynamicShadow = true;
	bCastCapsuleDirectShadow = true;
	bCastsDynamicIndirectShadow = true;
	bCastContactShadow = true;
	bCastDeepShadow;
	bCastHiddenShadow;
	bCastShadowAsTwoSided = true;
	bSelfShadowOnly;
//	bCastCinematicShadow;
//	bCastFarShadow;
	bLightAttachmentsAsGroup = true;
	bAffectDistanceFieldLighting = true;
	//	DynamicIndirectShadowMinVisibility = true;
	// Force inset shadows if capsule shadows are requested, as they can't be supported with full scene shadows
	bCastInsetShadow = true;
	DynamicIndirectShadowMinVisibility = 0.50f;
	DistanceFieldData = RenderData->LODResources[0].DistanceFieldData;
	CardRepresentationData = RenderData->LODResources[0].CardRepresentationData;


}

FAOStaticMeshSceneProxy::~FAOStaticMeshSceneProxy()
{
}

SIZE_T FAOStaticMeshSceneProxy::GetTypeHash() const
{
	static size_t UniquePointer;
	return reinterpret_cast<size_t>(&UniquePointer);
}

uint32 FAOStaticMeshSceneProxy::GetMemoryFootprint() const
{
	return (1024 * 1024);
}

bool FAOStaticMeshSceneProxy::HasDistanceFieldRepresentation() const
{
	return CastsDynamicShadow() && AffectsDistanceFieldLighting() && DistanceFieldData;
}

bool FAOStaticMeshSceneProxy::HasDynamicIndirectShadowCasterRepresentation() const
{
	return CastsDynamicShadow() && CastsDynamicIndirectShadow();
}


bool FAOStaticMeshSceneProxy::CanBeOccluded() const
{
	return !MaterialRelevance.bDisableDepthTest && !MaterialRelevance.bPostMotionBlurTranslucency && !ShouldRenderCustomDepth();
}

bool FAOStaticMeshSceneProxy::IsUsingDistanceCullFade() const
{
	return MaterialRelevance.bUsesDistanceCullFade;
}

void FAOStaticMeshSceneProxy::GetShadowShapes(FVector PreViewTranslation, TArray<FCapsuleShape3f>& OutCapsuleShapes) const
{
	if(AOStaticMeshComponent != nullptr) {
		FTransform ActorTransform = AOStaticMeshComponent->GetOwner()->GetActorTransform();
		FTransform ComponentTransform = AOStaticMeshComponent->GetComponentToWorld();
		FMatrix ReferenceToWorld = (ComponentTransform.ToMatrixWithScale());
		int32 CapsuleIndex = OutCapsuleShapes.Num();

		OutCapsuleShapes.SetNum(OutCapsuleShapes.Num() + AOStaticMeshComponent->PrimitiveAO.Num(), false);
		for(int32 i = 0; i < AOStaticMeshComponent->PrimitiveAO.Num(); ++i) {
			FVector4f Center = (FVector4f)(ReferenceToWorld.TransformPosition(AOStaticMeshComponent->PrimitiveAO[i].Center) + PreViewTranslation);
			FVector4f Orientation = (FVector4f)(ReferenceToWorld.TransformVector(AOStaticMeshComponent->PrimitiveAO[i].Rotation).GetSafeNormal());
			float Radius = AOStaticMeshComponent->PrimitiveAO[i].Radius;
			float Length = AOStaticMeshComponent->PrimitiveAO[i].Length;
			const float MaxScale = ReferenceToWorld.GetScaleVector().GetMax();
			{
				FCapsuleShape3f& NewCapsule = OutCapsuleShapes[CapsuleIndex++];
				NewCapsule.Center = Center;
				NewCapsule.Radius = Radius * MaxScale;
				if(AOStaticMeshComponent->PrimitiveAO[i].Type == EPrimitiveAOType::CAPSULE) {
					NewCapsule.Orientation = Orientation;
					NewCapsule.Length = Length * MaxScale;
				} else {
					NewCapsule.Orientation = FVector4f(0, 0, 0, 1);
					NewCapsule.Length = 0;
				}
			}
		}
	}
}