I have an open ended question about which testing framework to use for our unit tests and integration test. In our previous project, we used Spec test and it was working well but since CQTest was added in Unreal engine 5 we wanted to know which framework we should use on our new project.
Also, do you know how we could handle test flakiness caused by external plugins sending error logs which cause running test to fail?
I have an Idea for Spec test where I would implement FAutomationTestBase::ShouldCaptureLogCategory to whitelist only relevant log related to the test in our own BEGIN_DEFINE_SPEC macro.
CQTest is what we are using internally and we are actively porting older tests to that type.
The main selling points of CQTest are:
a more modern syntax
a revisited test interface
a stronger approach to async calls through the TestCommandBuilder
actively extended.
We don’t have a side by side example of Spec test vs CQ test.
But this example in Lyra sample project would be more verbose in Spec test:
TEST_CLASS_WITH_FLAGS(AbilitySpawnerMapTest, "Project.Functional Tests.ShooterTests.GameplayAbility", EAutomationTestFlags::EditorContext | EAutomationTestFlags::ProductFilter)
{
TUniquePtr<FMapTestSpawner> Spawner;
ALyraCharacter* Player{ nullptr };
AActor* GameplayEffectPad{ nullptr };
ULyraAbilitySystemComponent* AbilitySystemComponent{ nullptr };
const ULyraHealthSet* HealthSet{ nullptr };
// Fetches the GameplayEffect which will trigger and apply the damage specified to the player
void DoDamageToPlayer(double Damage)
{
const TSubclassOf<UGameplayEffect> DamageEffect = ULyraAssetManager::GetSubclass(ULyraGameData::Get().DamageGameplayEffect_SetByCaller);
FGameplayEffectSpecHandle SpecHandle = AbilitySystemComponent->MakeOutgoingSpec(DamageEffect, 1.0, AbilitySystemComponent->MakeEffectContext());
ASSERT_THAT(IsTrue(SpecHandle.IsValid()));
SpecHandle.Data->SetSetByCallerMagnitude(LyraGameplayTags::SetByCaller_Damage, Damage);
FActiveGameplayEffectHandle Handle = AbilitySystemComponent->ApplyGameplayEffectSpecToSelf(*SpecHandle.Data.Get());
ASSERT_THAT(IsTrue(Handle.WasSuccessfullyApplied()));
}
// Attempts to spawn a gameplay pad with a specified effect
void SpawnGameplayPad(const FString& EffectName)
{
UClass* DesiredEffect = CQTestAssetHelper::GetBlueprintClass(EffectName);
ASSERT_THAT(IsNotNull(DesiredEffect));
UClass* GameplayEffectPadBp = CQTestAssetHelper::GetBlueprintClass(TEXT("BP_GameplayEffectPad"));
ASSERT_THAT(IsNotNull(GameplayEffectPadBp));
GameplayEffectPad = &TObjectBuilder<AActor>(*Spawner, GameplayEffectPadBp)
.SetParam("GameplayEffectToApply", DesiredEffect)
.Spawn(Player->GetTransform());
ASSERT_THAT(IsNotNull(GameplayEffectPad));
}
// Checks to see if the player has been damaged
bool IsPlayerDamaged()
{
return HealthSet->GetHealth() < HealthSet->GetMaxHealth();
}
/**
* Run before each TEST_METHOD to load our level, initialize our Player, and Player components needed for the tests before to execute successfully.
* If an ASSERT_THAT fails at any point, the TEST_METHODS will also fail as this means that our test prerequisites were not setup
*/
BEFORE_EACH()
{
const FString LevelName = TEXT("L_ShooterTest_Basic");
TOptional<FString> PackagePath = CQTestAssetHelper::FindAssetPackagePathByName(LevelName);
ASSERT_THAT(IsTrue(PackagePath.IsSet(), "Could not find the level package."));
Spawner = MakeUnique<FMapTestSpawner>(PackagePath.GetValue(), LevelName);
Spawner->AddWaitUntilLoadedCommand(TestRunner);
const FTimespan LoadingScreenTimeout = FTimespan::FromSeconds(30);
TestCommandBuilder
.StartWhen([this]() { return nullptr != Spawner->FindFirstPlayerPawn(); }, LoadingScreenTimeout)
.Do([this]() {
Player = CastChecked<ALyraCharacter>(Spawner->FindFirstPlayerPawn());
AbilitySystemComponent = Player->GetLyraAbilitySystemComponent();
ASSERT_THAT(IsNotNull(AbilitySystemComponent));
HealthSet = AbilitySystemComponent->GetSetChecked<ULyraHealthSet>();
ASSERT_THAT(IsTrue(HealthSet->GetHealth() > 0));
ASSERT_THAT(IsTrue(HealthSet->GetHealth() == HealthSet->GetMaxHealth()));
});
}
// Tests to verify that the Player is able to get damaged
TEST_METHOD(PlayerOnDamageSpawner_Eventually_LosesHealth)
{
TestCommandBuilder
.StartWhen([this]() { return !AbilitySystemComponent->HasMatchingGameplayTag(TAG_Gameplay_DamageImmunity); })
.Then([this]() { SpawnGameplayPad(TEXT("GE_GameplayEffectPad_Damage")); })
.Until([this]() { return IsPlayerDamaged(); });
}
// Tests to verify that the Player is able to get recover after being damaged
TEST_METHOD(PlayerMissingHealth_OnHealSpawner_RestoresHealth)
{
TestCommandBuilder
.StartWhen([this]() { return !AbilitySystemComponent->HasMatchingGameplayTag(TAG_Gameplay_DamageImmunity); })
.Then([this]() { DoDamageToPlayer(10.0); })
.Until([this]() { return IsPlayerDamaged(); })
.Then([this]() { SpawnGameplayPad(TEXT("GE_GameplayEffectPad_Heal")); })
.Until([this]() { return !IsPlayerDamaged(); });
}
};
see file: Lyra/Plugins/GameFeatures/ShooterTests/Source/ShooterTestsRuntime/Private/ShooterTestsMapTests.cpp
Hi Jerome, I have a couple of questions based on the answer you provided Dominic:
What are the advantages of CQ tests over Spec tests?
Do you have concrete examples where a test written as a CQ test is better than the same test written with specs, in terms of readability or maintainability?
What is the long-term roadmap for UE’s C++ test frameworks (Automation, Spec, CQTest, LLT)? Which ones should new projects invest in today?